You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by lz...@apache.org on 2026/05/12 21:42:51 UTC
(iggy) branch feat/dotnet-memory-usage updated (b249c7245 -> f8d58858f)
This is an automated email from the ASF dual-hosted git repository.
lukaszzborek pushed a change to branch feat/dotnet-memory-usage
in repository https://gitbox.apache.org/repos/asf/iggy.git
discard b249c7245 refactor(csharp): format object initializers for improved readability in BinaryMapper
new f8d58858f refactor(csharp): format object initializers for improved readability in BinaryMapper
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b249c7245)
\
N -- N -- N refs/heads/feat/dotnet-memory-usage (f8d58858f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj | 1 +
1 file changed, 1 insertion(+)
(iggy) 01/01: refactor(csharp): format object initializers for improved readability in BinaryMapper
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lukaszzborek pushed a commit to branch feat/dotnet-memory-usage
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit f8d58858fe31c42b4b187a1a4ba70b3740708003
Author: Łukasz Zborek <lu...@gmail.com>
AuthorDate: Tue May 12 23:10:41 2026 +0200
refactor(csharp): format object initializers for improved readability in BinaryMapper
---
foreign/csharp/Directory.Packages.props | 53 +-
.../FetchMessagesTests.cs | 428 ++--
.../IggyTypedConsumerTests.cs | 544 ++--
.../Iggy_SDK.Tests.Integration.csproj | 89 +-
foreign/csharp/Iggy_SDK.sln | 1 -
.../Iggy_SDK/Consumers/DeserializedMessage.cs | 130 +-
foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs | 100 +-
.../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 500 ++--
foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 1090 ++++----
.../csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs | 386 +--
.../Iggy_SDK/Consumers/ReceivedRentedMessage.cs | 252 +-
.../csharp/Iggy_SDK/Contracts/MessageResponse.cs | 192 +-
.../Iggy_SDK/Contracts/PolledMessagesRental.cs | 142 +-
.../Iggy_SDK/Contracts/RentedMessageResponse.cs | 142 +-
.../Iggy_SDK/Encryption/AesMessageEncryptor.cs | 258 +-
.../Iggy_SDK/Encryption/IMessageEncryptor.cs | 80 +-
.../Iggy_SDK/Extensions/IggyClientExtension.cs | 148 +-
.../csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs | 170 +-
.../Implementations/HttpMessageStream.cs | 1724 ++++++-------
.../IggyClient/Implementations/TcpMessageStream.cs | 2634 ++++++++++----------
.../JsonConverters/MessageResponseConverter.cs | 212 +-
foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 56 +-
.../ConsumerTests/RentedConsumerTests.cs | 966 +++----
.../ConsumerTests/RentedTypedConsumerTests.cs | 366 +--
.../MapperTests/MessageResponseConverterTests.cs | 1 -
25 files changed, 5330 insertions(+), 5334 deletions(-)
diff --git a/foreign/csharp/Directory.Packages.props b/foreign/csharp/Directory.Packages.props
index cd888e5b8..b2dee5c2a 100644
--- a/foreign/csharp/Directory.Packages.props
+++ b/foreign/csharp/Directory.Packages.props
@@ -1,26 +1,27 @@
-<Project>
- <PropertyGroup>
- <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
- <CentralPackageTransitivePinningEnabled>false</CentralPackageTransitivePinningEnabled>
- </PropertyGroup>
- <ItemGroup>
- <PackageVersion Include="FluentAssertions" Version="6.12.0" />
- <PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.0" />
- <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
- <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
- <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="10.0.102" />
- <PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.3.2" />
- <PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="2.0.2" />
- <PackageVersion Include="Moq" Version="4.20.72" />
- <PackageVersion Include="Reqnroll.xunit.v3" Version="3.3.3" />
- <PackageVersion Include="Shouldly" Version="4.3.0" />
- <PackageVersion Include="System.IO.Hashing" Version="8.0.0" />
- <PackageVersion Include="Testcontainers" Version="4.10.0" />
- <PackageVersion Include="TUnit" Version="1.12.93" />
- <PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5">
- <PrivateAssets>all</PrivateAssets>
- <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
- </PackageVersion>
- <PackageVersion Include="xunit.v3.mtp-v2" Version="3.2.2" />
- </ItemGroup>
-</Project>
+<Project>
+ <PropertyGroup>
+ <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
+ <CentralPackageTransitivePinningEnabled>false</CentralPackageTransitivePinningEnabled>
+ </PropertyGroup>
+ <ItemGroup>
+ <PackageVersion Include="FluentAssertions" Version="6.12.0" />
+ <PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.0" />
+ <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
+ <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
+ <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="10.0.102" />
+ <PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.3.2" />
+ <PackageVersion Include="Microsoft.Testing.Extensions.Retry" Version="2.2.1" />
+ <PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="2.0.2" />
+ <PackageVersion Include="Moq" Version="4.20.72" />
+ <PackageVersion Include="Reqnroll.xunit.v3" Version="3.3.3" />
+ <PackageVersion Include="Shouldly" Version="4.3.0" />
+ <PackageVersion Include="System.IO.Hashing" Version="8.0.0" />
+ <PackageVersion Include="Testcontainers" Version="4.10.0" />
+ <PackageVersion Include="TUnit" Version="1.12.93" />
+ <PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5">
+ <PrivateAssets>all</PrivateAssets>
+ <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+ </PackageVersion>
+ <PackageVersion Include="xunit.v3.mtp-v2" Version="3.2.2" />
+ </ItemGroup>
+</Project>
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs
index 91768ada4..153e7cea6 100644
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs
@@ -1,214 +1,214 @@
-// // Licensed to the Apache Software Foundation (ASF) under one
-// // or more contributor license agreements. See the NOTICE file
-// // distributed with this work for additional information
-// // regarding copyright ownership. The ASF licenses this file
-// // to you under the Apache License, Version 2.0 (the
-// // "License"); you may not use this file except in compliance
-// // with the License. You may obtain a copy of the License at
-// //
-// // http://www.apache.org/licenses/LICENSE-2.0
-// //
-// // Unless required by applicable law or agreed to in writing,
-// // software distributed under the License is distributed on an
-// // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// // KIND, either express or implied. See the License for the
-// // specific language governing permissions and limitations
-// // under the License.
-
-using System.Text;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Enums;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.Headers;
-using Apache.Iggy.IggyClient;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Messages;
-using Apache.Iggy.Tests.Integrations.Fixtures;
-using Shouldly;
-using Partitioning = Apache.Iggy.Kinds.Partitioning;
-
-namespace Apache.Iggy.Tests.Integrations;
-
-public class FetchMessagesTests
-{
- private const int MessageCount = 20;
- private const string TopicName = "topic";
- private const string HeadersTopicName = "headers-topic";
-
- [ClassDataSource<IggyServerFixture>(Shared = SharedType.PerAssembly)]
- public required IggyServerFixture Fixture { get; init; }
-
- private async Task<(IIggyClient client, string streamName)> CreateStreamWithMessages(Protocol protocol)
- {
- var client = await Fixture.CreateAuthenticatedClient(protocol);
-
- var streamName = $"fetch-msg-{Guid.NewGuid():N}";
-
- await client.CreateStreamAsync(streamName);
- await client.CreateTopicAsync(Identifier.String(streamName), TopicName, 1);
- await client.CreateTopicAsync(Identifier.String(streamName), HeadersTopicName, 1);
-
- await client.SendMessagesAsync(Identifier.String(streamName),
- Identifier.String(TopicName), Partitioning.None(),
- CreateMessagesWithoutHeader(MessageCount));
-
- await client.SendMessagesAsync(Identifier.String(streamName),
- Identifier.String(HeadersTopicName), Partitioning.None(),
- CreateMessagesWithHeader(MessageCount));
-
- return (client, streamName);
- }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task PollMessages_WithNoHeaders_Should_PollMessages_Successfully(Protocol protocol)
- {
- var (client, streamName) = await CreateStreamWithMessages(protocol);
-
- var response = await client.PollMessagesAsync(new MessageFetchRequest
- {
- Count = 10,
- AutoCommit = true,
- Consumer = Consumer.New(1),
- PartitionId = 0,
- PollingStrategy = PollingStrategy.Next(),
- StreamId = Identifier.String(streamName),
- TopicId = Identifier.String(TopicName)
- });
-
- response.Messages.Count.ShouldBe(10);
- response.PartitionId.ShouldBe(0);
- response.CurrentOffset.ShouldBe(19u);
-
- foreach (var responseMessage in response.Messages)
- {
- responseMessage.UserHeaders.ShouldBeNull();
- responseMessage.Payload.ShouldNotBeNull();
- responseMessage.Payload.Length.ShouldBeGreaterThan(0);
- }
- }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task PollMessages_InvalidTopic_Should_Throw_InvalidResponse(Protocol protocol)
- {
- var (client, streamName) = await CreateStreamWithMessages(protocol);
-
- var invalidFetchRequest = new MessageFetchRequest
- {
- Count = 10,
- AutoCommit = true,
- Consumer = Consumer.New(1),
- PartitionId = 0,
- PollingStrategy = PollingStrategy.Next(),
- StreamId = Identifier.String(streamName),
- TopicId = Identifier.Numeric(2137)
- };
-
- await Should.ThrowAsync<IggyInvalidStatusCodeException>(() =>
- client.PollMessagesAsync(invalidFetchRequest));
- }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task PollMessages_WithHeaders_Should_PollMessages_Successfully(Protocol protocol)
- {
- var (client, streamName) = await CreateStreamWithMessages(protocol);
-
- var headersMessageFetchRequest = new MessageFetchRequest
- {
- Count = 10,
- AutoCommit = true,
- Consumer = Consumer.New(1),
- PartitionId = 0,
- PollingStrategy = PollingStrategy.Next(),
- StreamId = Identifier.String(streamName),
- TopicId = Identifier.String(HeadersTopicName)
- };
-
- var response = await client.PollMessagesAsync(headersMessageFetchRequest);
- response.Messages.Count.ShouldBe(10);
- response.PartitionId.ShouldBe(0);
- response.CurrentOffset.ShouldBe(19u);
- foreach (var responseMessage in response.Messages)
- {
- responseMessage.UserHeaders.ShouldNotBeNull();
- responseMessage.UserHeaders.Count.ShouldBe(2);
- responseMessage.UserHeaders[HeaderKey.FromString("header1")].ToString().ShouldBe("value1");
- responseMessage.UserHeaders[HeaderKey.FromString("header2")].ToInt32().ShouldBeGreaterThan(0);
- }
- }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task PollMessages_WithHeaders_Should_AutoParseUserHeadersFromRaw(Protocol protocol)
- {
- var (client, streamName) = await CreateStreamWithMessages(protocol);
-
- var response = await client.PollMessagesAsync(new MessageFetchRequest
- {
- Count = 1,
- AutoCommit = true,
- Consumer = Consumer.New(2),
- PartitionId = 0,
- PollingStrategy = PollingStrategy.First(),
- StreamId = Identifier.String(streamName),
- TopicId = Identifier.String(HeadersTopicName)
- });
-
- response.Messages.Count.ShouldBe(1);
- var msg = response.Messages[0];
-
- Dictionary<HeaderKey, HeaderValue>? parsed = msg.UserHeaders;
- parsed.ShouldNotBeNull();
- parsed!.Count.ShouldBe(2);
- parsed[HeaderKey.FromString("header1")].ToString().ShouldBe("value1");
- parsed[HeaderKey.FromString("header2")].ToInt32().ShouldBe(14);
-
- Dictionary<HeaderKey, HeaderValue>? second = msg.UserHeaders;
- second.ShouldBeSameAs(parsed);
- }
-
- private static Message[] CreateMessagesWithoutHeader(int count)
- {
- var messages = new List<Message>();
- for (var i = 0; i < count; i++)
- {
- var dummyJson = $$"""
- {
- "userId": {{i + 1}},
- "id": {{i + 1}},
- "title": "delete",
- "completed": false
- }
- """;
- messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson)));
- }
-
- return messages.ToArray();
- }
-
- private static Message[] CreateMessagesWithHeader(int count)
- {
- var messages = new List<Message>();
- for (var i = 0; i < count; i++)
- {
- var dummyJson = $$"""
- {
- "userId": {{i + 1}},
- "id": {{i + 1}},
- "title": "delete",
- "completed": false
- }
- """;
- messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson),
- new Dictionary<HeaderKey, HeaderValue>
- {
- { HeaderKey.FromString("header1"), HeaderValue.FromString("value1") },
- { HeaderKey.FromString("header2"), HeaderValue.FromInt32(14 + i) }
- }));
- }
-
- return messages.ToArray();
- }
-}
+// // Licensed to the Apache Software Foundation (ASF) under one
+// // or more contributor license agreements. See the NOTICE file
+// // distributed with this work for additional information
+// // regarding copyright ownership. The ASF licenses this file
+// // to you under the Apache License, Version 2.0 (the
+// // "License"); you may not use this file except in compliance
+// // with the License. You may obtain a copy of the License at
+// //
+// // http://www.apache.org/licenses/LICENSE-2.0
+// //
+// // Unless required by applicable law or agreed to in writing,
+// // software distributed under the License is distributed on an
+// // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// // KIND, either express or implied. See the License for the
+// // specific language governing permissions and limitations
+// // under the License.
+
+using System.Text;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Headers;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Messages;
+using Apache.Iggy.Tests.Integrations.Fixtures;
+using Shouldly;
+using Partitioning = Apache.Iggy.Kinds.Partitioning;
+
+namespace Apache.Iggy.Tests.Integrations;
+
+public class FetchMessagesTests
+{
+ private const int MessageCount = 20;
+ private const string TopicName = "topic";
+ private const string HeadersTopicName = "headers-topic";
+
+ [ClassDataSource<IggyServerFixture>(Shared = SharedType.PerAssembly)]
+ public required IggyServerFixture Fixture { get; init; }
+
+ private async Task<(IIggyClient client, string streamName)> CreateStreamWithMessages(Protocol protocol)
+ {
+ var client = await Fixture.CreateAuthenticatedClient(protocol);
+
+ var streamName = $"fetch-msg-{Guid.NewGuid():N}";
+
+ await client.CreateStreamAsync(streamName);
+ await client.CreateTopicAsync(Identifier.String(streamName), TopicName, 1);
+ await client.CreateTopicAsync(Identifier.String(streamName), HeadersTopicName, 1);
+
+ await client.SendMessagesAsync(Identifier.String(streamName),
+ Identifier.String(TopicName), Partitioning.None(),
+ CreateMessagesWithoutHeader(MessageCount));
+
+ await client.SendMessagesAsync(Identifier.String(streamName),
+ Identifier.String(HeadersTopicName), Partitioning.None(),
+ CreateMessagesWithHeader(MessageCount));
+
+ return (client, streamName);
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollMessages_WithNoHeaders_Should_PollMessages_Successfully(Protocol protocol)
+ {
+ var (client, streamName) = await CreateStreamWithMessages(protocol);
+
+ var response = await client.PollMessagesAsync(new MessageFetchRequest
+ {
+ Count = 10,
+ AutoCommit = true,
+ Consumer = Consumer.New(1),
+ PartitionId = 0,
+ PollingStrategy = PollingStrategy.Next(),
+ StreamId = Identifier.String(streamName),
+ TopicId = Identifier.String(TopicName)
+ });
+
+ response.Messages.Count.ShouldBe(10);
+ response.PartitionId.ShouldBe(0);
+ response.CurrentOffset.ShouldBe(19u);
+
+ foreach (var responseMessage in response.Messages)
+ {
+ responseMessage.UserHeaders.ShouldBeNull();
+ responseMessage.Payload.ShouldNotBeNull();
+ responseMessage.Payload.Length.ShouldBeGreaterThan(0);
+ }
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollMessages_InvalidTopic_Should_Throw_InvalidResponse(Protocol protocol)
+ {
+ var (client, streamName) = await CreateStreamWithMessages(protocol);
+
+ var invalidFetchRequest = new MessageFetchRequest
+ {
+ Count = 10,
+ AutoCommit = true,
+ Consumer = Consumer.New(1),
+ PartitionId = 0,
+ PollingStrategy = PollingStrategy.Next(),
+ StreamId = Identifier.String(streamName),
+ TopicId = Identifier.Numeric(2137)
+ };
+
+ await Should.ThrowAsync<IggyInvalidStatusCodeException>(() =>
+ client.PollMessagesAsync(invalidFetchRequest));
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollMessages_WithHeaders_Should_PollMessages_Successfully(Protocol protocol)
+ {
+ var (client, streamName) = await CreateStreamWithMessages(protocol);
+
+ var headersMessageFetchRequest = new MessageFetchRequest
+ {
+ Count = 10,
+ AutoCommit = true,
+ Consumer = Consumer.New(1),
+ PartitionId = 0,
+ PollingStrategy = PollingStrategy.Next(),
+ StreamId = Identifier.String(streamName),
+ TopicId = Identifier.String(HeadersTopicName)
+ };
+
+ var response = await client.PollMessagesAsync(headersMessageFetchRequest);
+ response.Messages.Count.ShouldBe(10);
+ response.PartitionId.ShouldBe(0);
+ response.CurrentOffset.ShouldBe(19u);
+ foreach (var responseMessage in response.Messages)
+ {
+ responseMessage.UserHeaders.ShouldNotBeNull();
+ responseMessage.UserHeaders.Count.ShouldBe(2);
+ responseMessage.UserHeaders[HeaderKey.FromString("header1")].ToString().ShouldBe("value1");
+ responseMessage.UserHeaders[HeaderKey.FromString("header2")].ToInt32().ShouldBeGreaterThan(0);
+ }
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollMessages_WithHeaders_Should_AutoParseUserHeadersFromRaw(Protocol protocol)
+ {
+ var (client, streamName) = await CreateStreamWithMessages(protocol);
+
+ var response = await client.PollMessagesAsync(new MessageFetchRequest
+ {
+ Count = 1,
+ AutoCommit = true,
+ Consumer = Consumer.New(2),
+ PartitionId = 0,
+ PollingStrategy = PollingStrategy.First(),
+ StreamId = Identifier.String(streamName),
+ TopicId = Identifier.String(HeadersTopicName)
+ });
+
+ response.Messages.Count.ShouldBe(1);
+ var msg = response.Messages[0];
+
+ Dictionary<HeaderKey, HeaderValue>? parsed = msg.UserHeaders;
+ parsed.ShouldNotBeNull();
+ parsed!.Count.ShouldBe(2);
+ parsed[HeaderKey.FromString("header1")].ToString().ShouldBe("value1");
+ parsed[HeaderKey.FromString("header2")].ToInt32().ShouldBe(14);
+
+ Dictionary<HeaderKey, HeaderValue>? second = msg.UserHeaders;
+ second.ShouldBeSameAs(parsed);
+ }
+
+ private static Message[] CreateMessagesWithoutHeader(int count)
+ {
+ var messages = new List<Message>();
+ for (var i = 0; i < count; i++)
+ {
+ var dummyJson = $$"""
+ {
+ "userId": {{i + 1}},
+ "id": {{i + 1}},
+ "title": "delete",
+ "completed": false
+ }
+ """;
+ messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson)));
+ }
+
+ return messages.ToArray();
+ }
+
+ private static Message[] CreateMessagesWithHeader(int count)
+ {
+ var messages = new List<Message>();
+ for (var i = 0; i < count; i++)
+ {
+ var dummyJson = $$"""
+ {
+ "userId": {{i + 1}},
+ "id": {{i + 1}},
+ "title": "delete",
+ "completed": false
+ }
+ """;
+ messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson),
+ new Dictionary<HeaderKey, HeaderValue>
+ {
+ { HeaderKey.FromString("header1"), HeaderValue.FromString("value1") },
+ { HeaderKey.FromString("header2"), HeaderValue.FromInt32(14 + i) }
+ }));
+ }
+
+ return messages.ToArray();
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs
index 8dd3160ff..eefbfd834 100644
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs
@@ -1,272 +1,272 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Text;
-using Apache.Iggy.Consumers;
-using Apache.Iggy.Enums;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.IggyClient;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Messages;
-using Apache.Iggy.Tests.Integrations.Fixtures;
-using Microsoft.Extensions.Logging.Abstractions;
-using Shouldly;
-using Partitioning = Apache.Iggy.Kinds.Partitioning;
-
-namespace Apache.Iggy.Tests.Integrations;
-
-public class IggyTypedConsumerTests
-{
- [ClassDataSource<IggyServerFixture>(Shared = SharedType.PerAssembly)]
- public required IggyServerFixture Fixture { get; init; }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task ReceiveRentedDeserializedAsync_Should_YieldMessages_WithCorrectData(Protocol protocol)
- {
- var client = protocol == Protocol.Tcp
- ? await Fixture.CreateTcpClient()
- : await Fixture.CreateHttpClient();
-
- var testStream = await CreateTestStreamWithMessages(client, protocol);
-
- var consumerId = protocol == Protocol.Tcp ? 200 : 300;
- IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
- AutoCommitMode.Disabled, new Utf8StringDeserializer());
-
- await consumer.InitAsync();
-
- var received = new List<DeserializedMessage<string>>();
- var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
-
- await foreach (DeserializedMessage<string> msg in consumer.ReceiveRentedDeserializedAsync(cts.Token))
- {
- msg.ShouldNotBeNull();
- msg.Status.ShouldBe(MessageStatus.Success);
- msg.Data.ShouldNotBeNull();
- msg.Data.ShouldStartWith("Test message");
- msg.PartitionId.ShouldBe(1u);
- msg.Error.ShouldBeNull();
- received.Add(msg);
- if (received.Count >= 10)
- {
- break;
- }
- }
-
- received.Count.ShouldBeGreaterThanOrEqualTo(10);
- await consumer.DisposeAsync();
- }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task ReceiveRentedDeserializedAsync_WithoutInit_Should_Throw_ConsumerNotInitializedException(
- Protocol protocol)
- {
- var client = protocol == Protocol.Tcp
- ? await Fixture.CreateTcpClient()
- : await Fixture.CreateHttpClient();
-
- var testStream = await CreateTestStreamWithMessages(client, protocol);
-
- var consumerId = protocol == Protocol.Tcp ? 201 : 301;
- IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
- AutoCommitMode.Disabled, new Utf8StringDeserializer());
-
- await Should.ThrowAsync<ConsumerNotInitializedException>(async () =>
- {
- await foreach (DeserializedMessage<string> _ in consumer.ReceiveRentedDeserializedAsync())
- {
- }
- });
-
- await consumer.DisposeAsync();
- }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task ReceiveRentedDeserializedAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Protocol protocol)
- {
- var client = protocol == Protocol.Tcp
- ? await Fixture.CreateTcpClient()
- : await Fixture.CreateHttpClient();
-
- var testStream = await CreateTestStreamWithMessages(client, protocol);
-
- var consumerId = protocol == Protocol.Tcp ? 202 : 302;
- IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
- AutoCommitMode.AfterReceive, new Utf8StringDeserializer(),
- PollingStrategy.First());
-
- await consumer.InitAsync();
-
- var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
- var count = 0;
-
- await foreach (DeserializedMessage<string> _ in consumer.ReceiveRentedDeserializedAsync(cts.Token))
- {
- count++;
- if (count >= 5)
- {
- break;
- }
- }
-
- await consumer.DisposeAsync();
-
- var offset = await client.GetOffsetAsync(Consumer.New(consumerId),
- Identifier.String(testStream.StreamId),
- Identifier.String(testStream.TopicId),
- 1u);
-
- offset.ShouldNotBeNull();
- offset.StoredOffset.ShouldBe(3ul);
- }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task ReceiveRentedDeserializedAsync_WithFailingDeserializer_Should_YieldDeserializationFailed(
- Protocol protocol)
- {
- var client = protocol == Protocol.Tcp
- ? await Fixture.CreateTcpClient()
- : await Fixture.CreateHttpClient();
-
- var testStream = await CreateTestStreamWithMessages(client, protocol);
-
- var consumerId = protocol == Protocol.Tcp ? 203 : 303;
- IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
- AutoCommitMode.Disabled, new FailingDeserializer());
-
- await consumer.InitAsync();
-
- var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
-
- await foreach (DeserializedMessage<string> msg in consumer.ReceiveRentedDeserializedAsync(cts.Token))
- {
- msg.Status.ShouldBe(MessageStatus.DeserializationFailed);
- msg.Data.ShouldBeNull();
- msg.Error.ShouldNotBeNull();
- msg.Error.ShouldBeOfType<InvalidOperationException>();
- break;
- }
-
- await consumer.DisposeAsync();
- }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task ReceiveRentedDeserializedAsync_Should_StopCleanly_OnCancellation(Protocol protocol)
- {
- var client = protocol == Protocol.Tcp
- ? await Fixture.CreateTcpClient()
- : await Fixture.CreateHttpClient();
-
- var testStream = await CreateTestStreamWithMessages(client, protocol);
-
- var consumerId = protocol == Protocol.Tcp ? 204 : 304;
- IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
- AutoCommitMode.Disabled, new Utf8StringDeserializer());
-
- await consumer.InitAsync();
-
- using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500));
-
- await Should.NotThrowAsync(async () =>
- {
- try
- {
- await foreach (DeserializedMessage<string> _ in consumer.ReceiveRentedDeserializedAsync(cts.Token))
- {
- }
- }
- catch (OperationCanceledException)
- {
- }
- });
-
- await consumer.DisposeAsync();
- }
-
- private static IggyConsumer<string> BuildTypedConsumer(IIggyClient client,
- TestStreamInfo stream,
- Consumer consumer,
- AutoCommitMode autoCommitMode,
- IDeserializer<string> deserializer,
- PollingStrategy? pollingStrategy = null)
- {
- var config = new IggyConsumerConfig<string>
- {
- StreamId = Identifier.String(stream.StreamId),
- TopicId = Identifier.String(stream.TopicId),
- Consumer = consumer,
- PollingStrategy = pollingStrategy ?? PollingStrategy.Next(),
- BatchSize = 10,
- PartitionId = 1,
- AutoCommitMode = autoCommitMode,
- AutoCommit = autoCommitMode != AutoCommitMode.Disabled,
- PollingIntervalMs = 0,
- Deserializer = deserializer
- };
- return new IggyConsumer<string>(client, config, NullLoggerFactory.Instance);
- }
-
- private async Task<TestStreamInfo> CreateTestStreamWithMessages(IIggyClient client, Protocol protocol,
- uint partitionsCount = 5, int messagesPerPartition = 100)
- {
- var streamId = $"typed_stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}";
- var topicId = "test_topic";
-
- await client.CreateStreamAsync(streamId);
- await client.CreateTopicAsync(Identifier.String(streamId), topicId, partitionsCount);
-
- for (uint partitionId = 0; partitionId < partitionsCount; partitionId++)
- {
- var messages = new List<Message>();
- for (var i = 0; i < messagesPerPartition; i++)
- {
- messages.Add(new Message(Guid.NewGuid(),
- Encoding.UTF8.GetBytes($"Test message {i} for partition {partitionId}")));
- }
-
- await client.SendMessagesAsync(Identifier.String(streamId),
- Identifier.String(topicId),
- Partitioning.PartitionId((int)partitionId),
- messages);
- }
-
- return new TestStreamInfo(streamId, topicId, partitionsCount, messagesPerPartition);
- }
-
- private record TestStreamInfo(string StreamId, string TopicId, uint PartitionsCount, int MessagesPerPartition);
-
- private sealed class Utf8StringDeserializer : IDeserializer<string>
- {
- public string Deserialize(ReadOnlyMemory<byte> data)
- {
- return Encoding.UTF8.GetString(data.Span);
- }
- }
-
- private sealed class FailingDeserializer : IDeserializer<string>
- {
- public string Deserialize(ReadOnlyMemory<byte> data)
- {
- throw new InvalidOperationException("Intentional deserialization failure");
- }
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Text;
+using Apache.Iggy.Consumers;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Messages;
+using Apache.Iggy.Tests.Integrations.Fixtures;
+using Microsoft.Extensions.Logging.Abstractions;
+using Shouldly;
+using Partitioning = Apache.Iggy.Kinds.Partitioning;
+
+namespace Apache.Iggy.Tests.Integrations;
+
+public class IggyTypedConsumerTests
+{
+ [ClassDataSource<IggyServerFixture>(Shared = SharedType.PerAssembly)]
+ public required IggyServerFixture Fixture { get; init; }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task ReceiveRentedDeserializedAsync_Should_YieldMessages_WithCorrectData(Protocol protocol)
+ {
+ var client = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient()
+ : await Fixture.CreateHttpClient();
+
+ var testStream = await CreateTestStreamWithMessages(client, protocol);
+
+ var consumerId = protocol == Protocol.Tcp ? 200 : 300;
+ IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
+ AutoCommitMode.Disabled, new Utf8StringDeserializer());
+
+ await consumer.InitAsync();
+
+ var received = new List<DeserializedMessage<string>>();
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+
+ await foreach (DeserializedMessage<string> msg in consumer.ReceiveRentedDeserializedAsync(cts.Token))
+ {
+ msg.ShouldNotBeNull();
+ msg.Status.ShouldBe(MessageStatus.Success);
+ msg.Data.ShouldNotBeNull();
+ msg.Data.ShouldStartWith("Test message");
+ msg.PartitionId.ShouldBe(1u);
+ msg.Error.ShouldBeNull();
+ received.Add(msg);
+ if (received.Count >= 10)
+ {
+ break;
+ }
+ }
+
+ received.Count.ShouldBeGreaterThanOrEqualTo(10);
+ await consumer.DisposeAsync();
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task ReceiveRentedDeserializedAsync_WithoutInit_Should_Throw_ConsumerNotInitializedException(
+ Protocol protocol)
+ {
+ var client = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient()
+ : await Fixture.CreateHttpClient();
+
+ var testStream = await CreateTestStreamWithMessages(client, protocol);
+
+ var consumerId = protocol == Protocol.Tcp ? 201 : 301;
+ IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
+ AutoCommitMode.Disabled, new Utf8StringDeserializer());
+
+ await Should.ThrowAsync<ConsumerNotInitializedException>(async () =>
+ {
+ await foreach (DeserializedMessage<string> _ in consumer.ReceiveRentedDeserializedAsync())
+ {
+ }
+ });
+
+ await consumer.DisposeAsync();
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task ReceiveRentedDeserializedAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Protocol protocol)
+ {
+ var client = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient()
+ : await Fixture.CreateHttpClient();
+
+ var testStream = await CreateTestStreamWithMessages(client, protocol);
+
+ var consumerId = protocol == Protocol.Tcp ? 202 : 302;
+ IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
+ AutoCommitMode.AfterReceive, new Utf8StringDeserializer(),
+ PollingStrategy.First());
+
+ await consumer.InitAsync();
+
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+ var count = 0;
+
+ await foreach (DeserializedMessage<string> _ in consumer.ReceiveRentedDeserializedAsync(cts.Token))
+ {
+ count++;
+ if (count >= 5)
+ {
+ break;
+ }
+ }
+
+ await consumer.DisposeAsync();
+
+ var offset = await client.GetOffsetAsync(Consumer.New(consumerId),
+ Identifier.String(testStream.StreamId),
+ Identifier.String(testStream.TopicId),
+ 1u);
+
+ offset.ShouldNotBeNull();
+ offset.StoredOffset.ShouldBe(3ul);
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task ReceiveRentedDeserializedAsync_WithFailingDeserializer_Should_YieldDeserializationFailed(
+ Protocol protocol)
+ {
+ var client = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient()
+ : await Fixture.CreateHttpClient();
+
+ var testStream = await CreateTestStreamWithMessages(client, protocol);
+
+ var consumerId = protocol == Protocol.Tcp ? 203 : 303;
+ IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
+ AutoCommitMode.Disabled, new FailingDeserializer());
+
+ await consumer.InitAsync();
+
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+
+ await foreach (DeserializedMessage<string> msg in consumer.ReceiveRentedDeserializedAsync(cts.Token))
+ {
+ msg.Status.ShouldBe(MessageStatus.DeserializationFailed);
+ msg.Data.ShouldBeNull();
+ msg.Error.ShouldNotBeNull();
+ msg.Error.ShouldBeOfType<InvalidOperationException>();
+ break;
+ }
+
+ await consumer.DisposeAsync();
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task ReceiveRentedDeserializedAsync_Should_StopCleanly_OnCancellation(Protocol protocol)
+ {
+ var client = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient()
+ : await Fixture.CreateHttpClient();
+
+ var testStream = await CreateTestStreamWithMessages(client, protocol);
+
+ var consumerId = protocol == Protocol.Tcp ? 204 : 304;
+ IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
+ AutoCommitMode.Disabled, new Utf8StringDeserializer());
+
+ await consumer.InitAsync();
+
+ using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500));
+
+ await Should.NotThrowAsync(async () =>
+ {
+ try
+ {
+ await foreach (DeserializedMessage<string> _ in consumer.ReceiveRentedDeserializedAsync(cts.Token))
+ {
+ }
+ }
+ catch (OperationCanceledException)
+ {
+ }
+ });
+
+ await consumer.DisposeAsync();
+ }
+
+ private static IggyConsumer<string> BuildTypedConsumer(IIggyClient client,
+ TestStreamInfo stream,
+ Consumer consumer,
+ AutoCommitMode autoCommitMode,
+ IDeserializer<string> deserializer,
+ PollingStrategy? pollingStrategy = null)
+ {
+ var config = new IggyConsumerConfig<string>
+ {
+ StreamId = Identifier.String(stream.StreamId),
+ TopicId = Identifier.String(stream.TopicId),
+ Consumer = consumer,
+ PollingStrategy = pollingStrategy ?? PollingStrategy.Next(),
+ BatchSize = 10,
+ PartitionId = 1,
+ AutoCommitMode = autoCommitMode,
+ AutoCommit = autoCommitMode != AutoCommitMode.Disabled,
+ PollingIntervalMs = 0,
+ Deserializer = deserializer
+ };
+ return new IggyConsumer<string>(client, config, NullLoggerFactory.Instance);
+ }
+
+ private async Task<TestStreamInfo> CreateTestStreamWithMessages(IIggyClient client, Protocol protocol,
+ uint partitionsCount = 5, int messagesPerPartition = 100)
+ {
+ var streamId = $"typed_stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}";
+ var topicId = "test_topic";
+
+ await client.CreateStreamAsync(streamId);
+ await client.CreateTopicAsync(Identifier.String(streamId), topicId, partitionsCount);
+
+ for (uint partitionId = 0; partitionId < partitionsCount; partitionId++)
+ {
+ var messages = new List<Message>();
+ for (var i = 0; i < messagesPerPartition; i++)
+ {
+ messages.Add(new Message(Guid.NewGuid(),
+ Encoding.UTF8.GetBytes($"Test message {i} for partition {partitionId}")));
+ }
+
+ await client.SendMessagesAsync(Identifier.String(streamId),
+ Identifier.String(topicId),
+ Partitioning.PartitionId((int)partitionId),
+ messages);
+ }
+
+ return new TestStreamInfo(streamId, topicId, partitionsCount, messagesPerPartition);
+ }
+
+ private record TestStreamInfo(string StreamId, string TopicId, uint PartitionsCount, int MessagesPerPartition);
+
+ private sealed class Utf8StringDeserializer : IDeserializer<string>
+ {
+ public string Deserialize(ReadOnlyMemory<byte> data)
+ {
+ return Encoding.UTF8.GetString(data.Span);
+ }
+ }
+
+ private sealed class FailingDeserializer : IDeserializer<string>
+ {
+ public string Deserialize(ReadOnlyMemory<byte> data)
+ {
+ throw new InvalidOperationException("Intentional deserialization failure");
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj b/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj
index a0bc77def..1b229b82c 100644
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj
@@ -1,44 +1,45 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
- <PropertyGroup>
- <ImplicitUsings>enable</ImplicitUsings>
- <Nullable>enable</Nullable>
- <OutputType>Exe</OutputType>
- <TargetFrameworks>net8.0;net10.0</TargetFrameworks>
- <AssemblyName>Apache.Iggy.Tests.Integrations</AssemblyName>
- <RootNamespace>Apache.Iggy.Tests.Integrations</RootNamespace>
- <IsTestingPlatformApplication>true</IsTestingPlatformApplication>
- </PropertyGroup>
-
- <ItemGroup>
- <PackageReference Include="Microsoft.Testing.Extensions.CodeCoverage" />
- <PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
- <PackageReference Include="Shouldly" />
- <PackageReference Include="Testcontainers" />
- <PackageReference Include="TUnit" />
- </ItemGroup>
-
- <ItemGroup>
- <ProjectReference Include="..\Iggy_SDK\Iggy_SDK.csproj" />
- </ItemGroup>
-
- <ItemGroup>
- <Content Include="..\..\..\core\certs\iggy.pfx">
- <Link>Certs\iggy.pfx</Link>
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </Content>
- <Content Include="..\..\..\core\certs\iggy_ca_cert.pem">
- <Link>Certs\iggy_ca_cert.pem</Link>
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </Content>
- <Content Include="..\..\..\core\certs\iggy_cert.pem">
- <Link>Certs\iggy_cert.pem</Link>
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </Content>
- <Content Include="..\..\..\core\certs\iggy_key.pem">
- <Link>Certs\iggy_key.pem</Link>
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </Content>
- </ItemGroup>
-
-</Project>
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <ImplicitUsings>enable</ImplicitUsings>
+ <Nullable>enable</Nullable>
+ <OutputType>Exe</OutputType>
+ <TargetFrameworks>net8.0;net10.0</TargetFrameworks>
+ <AssemblyName>Apache.Iggy.Tests.Integrations</AssemblyName>
+ <RootNamespace>Apache.Iggy.Tests.Integrations</RootNamespace>
+ <IsTestingPlatformApplication>true</IsTestingPlatformApplication>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="Microsoft.Testing.Extensions.CodeCoverage" />
+ <PackageReference Include="Microsoft.Testing.Extensions.Retry" />
+ <PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
+ <PackageReference Include="Shouldly" />
+ <PackageReference Include="Testcontainers" />
+ <PackageReference Include="TUnit" />
+ </ItemGroup>
+
+ <ItemGroup>
+ <ProjectReference Include="..\Iggy_SDK\Iggy_SDK.csproj" />
+ </ItemGroup>
+
+ <ItemGroup>
+ <Content Include="..\..\..\core\certs\iggy.pfx">
+ <Link>Certs\iggy.pfx</Link>
+ <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ </Content>
+ <Content Include="..\..\..\core\certs\iggy_ca_cert.pem">
+ <Link>Certs\iggy_ca_cert.pem</Link>
+ <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ </Content>
+ <Content Include="..\..\..\core\certs\iggy_cert.pem">
+ <Link>Certs\iggy_cert.pem</Link>
+ <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ </Content>
+ <Content Include="..\..\..\core\certs\iggy_key.pem">
+ <Link>Certs\iggy_key.pem</Link>
+ <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ </Content>
+ </ItemGroup>
+
+</Project>
diff --git a/foreign/csharp/Iggy_SDK.sln b/foreign/csharp/Iggy_SDK.sln
index 5732f6978..0a28df793 100644
--- a/foreign/csharp/Iggy_SDK.sln
+++ b/foreign/csharp/Iggy_SDK.sln
@@ -1,4 +1,3 @@
-
Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Iggy_SDK", "Iggy_SDK\Iggy_SDK.csproj", "{661540EB-81F9-492C-828B-CF787BEBD50B}"
EndProject
diff --git a/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs
index b7e8cdbb8..54c77ac74 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs
@@ -1,65 +1,65 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using Apache.Iggy.Headers;
-using Apache.Iggy.Messages;
-
-namespace Apache.Iggy.Consumers;
-
-/// <summary>
-/// Represents a message whose payload was deserialized directly from rented memory. The rented buffer has
-/// already been returned to the pool by the time this message is yielded, so only the header, user headers,
-/// and the deserialized <see cref="Data" /> are available; the raw payload bytes are not retained.
-/// </summary>
-/// <typeparam name="T">The deserialized payload type.</typeparam>
-public sealed class DeserializedMessage<T>
-{
- /// <summary>
- /// Message header.
- /// </summary>
- public required MessageHeader Header { get; init; }
-
- /// <summary>
- /// The deserialized payload. Null if <see cref="Status" /> is not <see cref="MessageStatus.Success" />.
- /// </summary>
- public T? Data { get; init; }
-
- /// <summary>
- /// Parsed user headers, if present.
- /// </summary>
- public Dictionary<HeaderKey, HeaderValue>? UserHeaders { get; init; }
-
- /// <summary>
- /// The current offset of this message in the partition.
- /// </summary>
- public required ulong CurrentOffset { get; init; }
-
- /// <summary>
- /// The partition ID from which this message was consumed.
- /// </summary>
- public uint PartitionId { get; init; }
-
- /// <summary>
- /// The status of the message (Success, DecryptionFailed, DeserializationFailed).
- /// </summary>
- public MessageStatus Status { get; init; } = MessageStatus.Success;
-
- /// <summary>
- /// The exception that occurred during processing, if any.
- /// </summary>
- public Exception? Error { get; init; }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Headers;
+using Apache.Iggy.Messages;
+
+namespace Apache.Iggy.Consumers;
+
+/// <summary>
+/// Represents a message whose payload was deserialized directly from rented memory. The rented buffer has
+/// already been returned to the pool by the time this message is yielded, so only the header, user headers,
+/// and the deserialized <see cref="Data" /> are available; the raw payload bytes are not retained.
+/// </summary>
+/// <typeparam name="T">The deserialized payload type.</typeparam>
+public sealed class DeserializedMessage<T>
+{
+ /// <summary>
+ /// Message header.
+ /// </summary>
+ public required MessageHeader Header { get; init; }
+
+ /// <summary>
+ /// The deserialized payload. Null if <see cref="Status" /> is not <see cref="MessageStatus.Success" />.
+ /// </summary>
+ public T? Data { get; init; }
+
+ /// <summary>
+ /// Parsed user headers, if present.
+ /// </summary>
+ public Dictionary<HeaderKey, HeaderValue>? UserHeaders { get; init; }
+
+ /// <summary>
+ /// The current offset of this message in the partition.
+ /// </summary>
+ public required ulong CurrentOffset { get; init; }
+
+ /// <summary>
+ /// The partition ID from which this message was consumed.
+ /// </summary>
+ public uint PartitionId { get; init; }
+
+ /// <summary>
+ /// The status of the message (Success, DecryptionFailed, DeserializationFailed).
+ /// </summary>
+ public MessageStatus Status { get; init; } = MessageStatus.Success;
+
+ /// <summary>
+ /// The exception that occurred during processing, if any.
+ /// </summary>
+ public Exception? Error { get; init; }
+}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs b/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs
index e22372144..041b75cb3 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs
@@ -1,50 +1,50 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-namespace Apache.Iggy.Consumers;
-
-/// <summary>
-/// Interface for deserializing message payloads from byte arrays to type T.
-/// <para>
-/// No type constraints are enforced on T to provide maximum flexibility.
-/// Implementations are responsible for ensuring that the provided byte data can be properly deserialized to the
-/// target type.
-/// </para>
-/// </summary>
-/// <typeparam name="T">
-/// The target type for deserialization. Can be any type - reference or value type, nullable or non-nullable.
-/// The deserializer implementation must be able to produce instances of the specific type.
-/// </typeparam>
-/// <remarks>
-/// Implementations should throw appropriate exceptions (e.g., <see cref="System.FormatException" />,
-/// <see cref="System.ArgumentException" />, or <see cref="System.InvalidOperationException" />)
-/// if the provided data cannot be deserialized to type T. These exceptions will be caught and logged by
-/// <see cref="IggyConsumer{T}" /> during message processing.
-/// </remarks>
-public interface IDeserializer<out T>
-{
- /// <summary>
- /// Deserializes a read-only memory into an instance of type T. Callers may pass a <c>byte[]</c> directly
- /// thanks to the implicit conversion to <see cref="ReadOnlyMemory{T}" />.
- /// </summary>
- /// <param name="data">
- /// Read-only memory containing the serialized data. The implementation MUST NOT retain a reference to
- /// the span after returning.
- /// </param>
- /// <returns>An instance of type T representing the deserialized data.</returns>
- T Deserialize(ReadOnlyMemory<byte> data);
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.Iggy.Consumers;
+
+/// <summary>
+/// Interface for deserializing message payloads from byte arrays to type T.
+/// <para>
+/// No type constraints are enforced on T to provide maximum flexibility.
+/// Implementations are responsible for ensuring that the provided byte data can be properly deserialized to the
+/// target type.
+/// </para>
+/// </summary>
+/// <typeparam name="T">
+/// The target type for deserialization. Can be any type - reference or value type, nullable or non-nullable.
+/// The deserializer implementation must be able to produce instances of the specific type.
+/// </typeparam>
+/// <remarks>
+/// Implementations should throw appropriate exceptions (e.g., <see cref="System.FormatException" />,
+/// <see cref="System.ArgumentException" />, or <see cref="System.InvalidOperationException" />)
+/// if the provided data cannot be deserialized to type T. These exceptions will be caught and logged by
+/// <see cref="IggyConsumer{T}" /> during message processing.
+/// </remarks>
+public interface IDeserializer<out T>
+{
+ /// <summary>
+ /// Deserializes a read-only memory into an instance of type T. Callers may pass a <c>byte[]</c> directly
+ /// thanks to the implicit conversion to <see cref="ReadOnlyMemory{T}" />.
+ /// </summary>
+ /// <param name="data">
+ /// Read-only memory containing the serialized data. The implementation MUST NOT retain a reference to
+ /// the span after returning.
+ /// </param>
+ /// <returns>An instance of type T representing the deserialized data.</returns>
+ T Deserialize(ReadOnlyMemory<byte> data);
+}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs
index a18f4c784..cbc34951f 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs
@@ -1,250 +1,250 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Runtime.CompilerServices;
-using System.Threading.Channels;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Enums;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.Headers;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Mappers;
-using Microsoft.Extensions.Logging;
-
-namespace Apache.Iggy.Consumers;
-
-public partial class IggyConsumer
-{
- private readonly Channel<ReceivedRentedMessage> _rentedChannel = Channel.CreateUnbounded<ReceivedRentedMessage>();
-
- /// <summary>
- /// Receives messages asynchronously as an async stream of rented messages. Each yielded
- /// <see cref="ReceivedRentedMessage" /> shares its underlying pooled buffer with the other messages from the
- /// same poll and MUST be disposed by the caller when processing is complete. The buffer is returned to the
- /// pool once every message of its batch has been disposed.
- /// </summary>
- /// <param name="ct">Cancellation token to stop receiving messages.</param>
- /// <returns>An async enumerable of rented messages.</returns>
- /// <exception cref="ConsumerNotInitializedException">Thrown when <see cref="InitAsync" /> has not been called.</exception>
- public async IAsyncEnumerable<ReceivedRentedMessage> ReceiveRentedAsync(
- [EnumeratorCancellation] CancellationToken ct = default)
- {
- if (!_isInitialized)
- {
- throw new ConsumerNotInitializedException();
- }
-
- do
- {
- if (!_rentedChannel.Reader.TryRead(out var message))
- {
- await PollRentedMessagesAsync(ct);
- continue;
- }
-
- yield return message;
-
- if (_config.AutoCommitMode == AutoCommitMode.AfterReceive)
- {
- await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct);
- }
- } while (!ct.IsCancellationRequested);
- }
-
- /// <summary>
- /// Publishes a single rented message from a polled batch to the consumer channel. Called once per
- /// message during <see cref="PollRentedMessagesAsync" />. The caller acquires a reference on
- /// <paramref name="rental" /> before invocation; the channel reader is expected to release that
- /// reference by disposing the produced <see cref="ReceivedRentedMessage" />. Override to redirect
- /// rented batches to a different sink (e.g. typed deserialization) — overrides must ensure the
- /// acquired reference is released exactly once on every path.
- /// </summary>
- /// <param name="rental">Reference-counted handle around the polled batch. Caller has already acquired one reference.</param>
- /// <param name="message">
- /// The rented message to publish. Payload and raw headers are slices of pooled memory tied to
- /// <paramref name="rental" />.
- /// </param>
- /// <param name="partitionId">Partition the message was polled from.</param>
- /// <param name="status">Outcome of any prior processing (e.g. decryption).</param>
- /// <param name="error">Exception captured if <paramref name="status" /> is non-success; otherwise null.</param>
- /// <param name="ct">Cancellation token.</param>
- protected virtual async Task PublishRentedAsync(RentedBatchHandle rental,
- RentedMessageResponse message,
- uint partitionId,
- MessageStatus status,
- Exception? error,
- CancellationToken ct)
- {
- await _rentedChannel.Writer.WriteAsync(new ReceivedRentedMessage
- {
- Handle = rental,
- Message = message,
- CurrentOffset = message.Header.Offset,
- PartitionId = partitionId,
- Status = status,
- Error = error
- }, ct);
- }
-
- /// <summary>
- /// Polls a rented batch from the server and publishes it via <see cref="PublishRentedAsync" />.
- /// Handles decryption, offset tracking, and auto-commit logic. Rental lifetime is managed via
- /// <see cref="RentedBatchHandle" /> shared by every produced message.
- /// </summary>
- protected async Task PollRentedMessagesAsync(CancellationToken ct)
- {
- if (!_joinedConsumerGroup)
- {
- LogConsumerGroupNotJoinedYetSkippingPolling();
- return;
- }
-
- await _pollingSemaphore.WaitAsync(ct);
-
- PolledMessagesRental? rental = null;
- RentedBatchHandle? batchHandle = null;
- try
- {
- if (_config.PollingIntervalMs > 0)
- {
- await WaitBeforePollingAsync(ct);
- }
-
- rental = await _client.PollMessagesRentedAsync(_config.StreamId, _config.TopicId,
- _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize,
- _config.AutoCommit, ct);
-
- if (rental.Messages.Count == 0)
- {
- if (_logger.IsEnabled(LogLevel.Debug))
- {
- _logger.LogDebug("No messages received from poll for partition {PartitionId}", rental.PartitionId);
- }
-
- return;
- }
-
- var partitionId = (uint)rental.PartitionId;
-
- var hasLastOffset = _lastPolledOffset.TryGetValue(rental.PartitionId, out var lastPolledPartitionOffset);
-
- var currentOffset = 0ul;
-
- batchHandle = new RentedBatchHandle(rental);
- var anyNewMessages = false;
- foreach (var message in rental.Messages)
- {
- if (hasLastOffset && message.Header.Offset <= lastPolledPartitionOffset)
- {
- continue;
- }
-
- var processedMessage = message;
- var status = MessageStatus.Success;
- Exception? error = null;
-
- if (_config.MessageEncryptor != null)
- {
- try
- {
- var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload.Span);
-
- Dictionary<HeaderKey, HeaderValue>? decryptedHeaders = null;
- if (!message.RawUserHeaders.IsEmpty)
- {
- var decryptedHeaderBytes =
- _config.MessageEncryptor.Decrypt(message.RawUserHeaders.Span);
- decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes);
- }
-
- processedMessage = new RentedMessageResponse
- {
- Header = message.Header,
- Payload = decryptedPayload,
- RawUserHeaders = ReadOnlyMemory<byte>.Empty,
- UserHeaders = decryptedHeaders
- };
- }
- catch (Exception ex)
- {
- LogFailedToDecryptMessage(ex, message.Header.Offset);
- status = MessageStatus.DecryptionFailed;
- error = ex;
- }
- }
-
- currentOffset = message.Header.Offset;
- batchHandle.Acquire();
- try
- {
- await PublishRentedAsync(batchHandle, processedMessage, partitionId, status, error, ct);
- }
- catch
- {
- batchHandle.Release();
- throw;
- }
-
- anyNewMessages = true;
- }
-
- if (!anyNewMessages
- && _config.AutoCommitMode != AutoCommitMode.Disabled)
- {
- if (_logger.IsEnabled(LogLevel.Debug))
- {
- _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}",
- lastPolledPartitionOffset, rental.PartitionId);
- }
-
- await StoreOffsetAsync(lastPolledPartitionOffset, partitionId, false, ct);
- }
-
- if (anyNewMessages)
- {
- _lastPolledOffset.AddOrUpdate(rental.PartitionId, currentOffset, (_, _) => currentOffset);
- }
-
- if (_config.PollingStrategy.Kind == MessagePolling.Offset)
- {
- _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1);
- }
- }
- catch (OperationCanceledException) when (ct.IsCancellationRequested)
- {
- throw;
- }
- catch (Exception ex)
- {
- LogFailedToPollMessages(ex);
- _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages"));
- }
- finally
- {
- if (batchHandle is not null)
- {
- batchHandle.Release();
- }
- else
- {
- rental?.Dispose();
- }
-
- _pollingSemaphore.Release();
- }
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Runtime.CompilerServices;
+using System.Threading.Channels;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Headers;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Mappers;
+using Microsoft.Extensions.Logging;
+
+namespace Apache.Iggy.Consumers;
+
+public partial class IggyConsumer
+{
+ private readonly Channel<ReceivedRentedMessage> _rentedChannel = Channel.CreateUnbounded<ReceivedRentedMessage>();
+
+ /// <summary>
+ /// Receives messages asynchronously as an async stream of rented messages. Each yielded
+ /// <see cref="ReceivedRentedMessage" /> shares its underlying pooled buffer with the other messages from the
+ /// same poll and MUST be disposed by the caller when processing is complete. The buffer is returned to the
+ /// pool once every message of its batch has been disposed.
+ /// </summary>
+ /// <param name="ct">Cancellation token to stop receiving messages.</param>
+ /// <returns>An async enumerable of rented messages.</returns>
+ /// <exception cref="ConsumerNotInitializedException">Thrown when <see cref="InitAsync" /> has not been called.</exception>
+ public async IAsyncEnumerable<ReceivedRentedMessage> ReceiveRentedAsync(
+ [EnumeratorCancellation] CancellationToken ct = default)
+ {
+ if (!_isInitialized)
+ {
+ throw new ConsumerNotInitializedException();
+ }
+
+ do
+ {
+ if (!_rentedChannel.Reader.TryRead(out var message))
+ {
+ await PollRentedMessagesAsync(ct);
+ continue;
+ }
+
+ yield return message;
+
+ if (_config.AutoCommitMode == AutoCommitMode.AfterReceive)
+ {
+ await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct);
+ }
+ } while (!ct.IsCancellationRequested);
+ }
+
+ /// <summary>
+ /// Publishes a single rented message from a polled batch to the consumer channel. Called once per
+ /// message during <see cref="PollRentedMessagesAsync" />. The caller acquires a reference on
+ /// <paramref name="rental" /> before invocation; the channel reader is expected to release that
+ /// reference by disposing the produced <see cref="ReceivedRentedMessage" />. Override to redirect
+ /// rented batches to a different sink (e.g. typed deserialization) — overrides must ensure the
+ /// acquired reference is released exactly once on every path.
+ /// </summary>
+ /// <param name="rental">Reference-counted handle around the polled batch. Caller has already acquired one reference.</param>
+ /// <param name="message">
+ /// The rented message to publish. Payload and raw headers are slices of pooled memory tied to
+ /// <paramref name="rental" />.
+ /// </param>
+ /// <param name="partitionId">Partition the message was polled from.</param>
+ /// <param name="status">Outcome of any prior processing (e.g. decryption).</param>
+ /// <param name="error">Exception captured if <paramref name="status" /> is non-success; otherwise null.</param>
+ /// <param name="ct">Cancellation token.</param>
+ protected virtual async Task PublishRentedAsync(RentedBatchHandle rental,
+ RentedMessageResponse message,
+ uint partitionId,
+ MessageStatus status,
+ Exception? error,
+ CancellationToken ct)
+ {
+ await _rentedChannel.Writer.WriteAsync(new ReceivedRentedMessage
+ {
+ Handle = rental,
+ Message = message,
+ CurrentOffset = message.Header.Offset,
+ PartitionId = partitionId,
+ Status = status,
+ Error = error
+ }, ct);
+ }
+
+ /// <summary>
+ /// Polls a rented batch from the server and publishes it via <see cref="PublishRentedAsync" />.
+ /// Handles decryption, offset tracking, and auto-commit logic. Rental lifetime is managed via
+ /// <see cref="RentedBatchHandle" /> shared by every produced message.
+ /// </summary>
+ protected async Task PollRentedMessagesAsync(CancellationToken ct)
+ {
+ if (!_joinedConsumerGroup)
+ {
+ LogConsumerGroupNotJoinedYetSkippingPolling();
+ return;
+ }
+
+ await _pollingSemaphore.WaitAsync(ct);
+
+ PolledMessagesRental? rental = null;
+ RentedBatchHandle? batchHandle = null;
+ try
+ {
+ if (_config.PollingIntervalMs > 0)
+ {
+ await WaitBeforePollingAsync(ct);
+ }
+
+ rental = await _client.PollMessagesRentedAsync(_config.StreamId, _config.TopicId,
+ _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize,
+ _config.AutoCommit, ct);
+
+ if (rental.Messages.Count == 0)
+ {
+ if (_logger.IsEnabled(LogLevel.Debug))
+ {
+ _logger.LogDebug("No messages received from poll for partition {PartitionId}", rental.PartitionId);
+ }
+
+ return;
+ }
+
+ var partitionId = (uint)rental.PartitionId;
+
+ var hasLastOffset = _lastPolledOffset.TryGetValue(rental.PartitionId, out var lastPolledPartitionOffset);
+
+ var currentOffset = 0ul;
+
+ batchHandle = new RentedBatchHandle(rental);
+ var anyNewMessages = false;
+ foreach (var message in rental.Messages)
+ {
+ if (hasLastOffset && message.Header.Offset <= lastPolledPartitionOffset)
+ {
+ continue;
+ }
+
+ var processedMessage = message;
+ var status = MessageStatus.Success;
+ Exception? error = null;
+
+ if (_config.MessageEncryptor != null)
+ {
+ try
+ {
+ var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload.Span);
+
+ Dictionary<HeaderKey, HeaderValue>? decryptedHeaders = null;
+ if (!message.RawUserHeaders.IsEmpty)
+ {
+ var decryptedHeaderBytes =
+ _config.MessageEncryptor.Decrypt(message.RawUserHeaders.Span);
+ decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes);
+ }
+
+ processedMessage = new RentedMessageResponse
+ {
+ Header = message.Header,
+ Payload = decryptedPayload,
+ RawUserHeaders = ReadOnlyMemory<byte>.Empty,
+ UserHeaders = decryptedHeaders
+ };
+ }
+ catch (Exception ex)
+ {
+ LogFailedToDecryptMessage(ex, message.Header.Offset);
+ status = MessageStatus.DecryptionFailed;
+ error = ex;
+ }
+ }
+
+ currentOffset = message.Header.Offset;
+ batchHandle.Acquire();
+ try
+ {
+ await PublishRentedAsync(batchHandle, processedMessage, partitionId, status, error, ct);
+ }
+ catch
+ {
+ batchHandle.Release();
+ throw;
+ }
+
+ anyNewMessages = true;
+ }
+
+ if (!anyNewMessages
+ && _config.AutoCommitMode != AutoCommitMode.Disabled)
+ {
+ if (_logger.IsEnabled(LogLevel.Debug))
+ {
+ _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}",
+ lastPolledPartitionOffset, rental.PartitionId);
+ }
+
+ await StoreOffsetAsync(lastPolledPartitionOffset, partitionId, false, ct);
+ }
+
+ if (anyNewMessages)
+ {
+ _lastPolledOffset.AddOrUpdate(rental.PartitionId, currentOffset, (_, _) => currentOffset);
+ }
+
+ if (_config.PollingStrategy.Kind == MessagePolling.Offset)
+ {
+ _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1);
+ }
+ }
+ catch (OperationCanceledException) when (ct.IsCancellationRequested)
+ {
+ throw;
+ }
+ catch (Exception ex)
+ {
+ LogFailedToPollMessages(ex);
+ _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages"));
+ }
+ finally
+ {
+ if (batchHandle is not null)
+ {
+ batchHandle.Release();
+ }
+ else
+ {
+ rental?.Dispose();
+ }
+
+ _pollingSemaphore.Release();
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
index 47349f9cf..504fa2f18 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
@@ -1,545 +1,545 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Collections.Concurrent;
-using System.Runtime.CompilerServices;
-using System.Threading.Channels;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Enums;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.Headers;
-using Apache.Iggy.IggyClient;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Mappers;
-using Apache.Iggy.Utils;
-using Microsoft.Extensions.Logging;
-
-namespace Apache.Iggy.Consumers;
-
-/// <summary>
-/// High-level consumer for receiving messages from Iggy streams.
-/// Provides automatic polling, offset management, and consumer group support.
-/// </summary>
-public partial class IggyConsumer : IAsyncDisposable
-{
- private readonly Channel<ReceivedMessage> _channel;
- private readonly IIggyClient _client;
- private readonly IggyConsumerConfig _config;
- private readonly SemaphoreSlim _connectionStateSemaphore = new(1, 1);
- private readonly EventAggregator<ConsumerErrorEventArgs> _consumerErrorEvents;
- private readonly ConcurrentDictionary<int, ulong> _lastPolledOffset = new();
- private readonly ILogger<IggyConsumer> _logger;
- private readonly SemaphoreSlim _pollingSemaphore = new(1, 1);
- private string? _consumerGroupName;
- private int _disposeState;
- private volatile bool _isInitialized;
- private volatile bool _joinedConsumerGroup;
- private long _lastPolledAtMs;
-
- /// <summary>Whether this consumer has been initialized via <see cref="InitAsync" />.</summary>
- protected bool IsInitialized => _isInitialized;
-
- /// <summary>
- /// Initializes a new instance of the <see cref="IggyConsumer" /> class
- /// </summary>
- /// <param name="client">The Iggy client for server communication</param>
- /// <param name="config">Consumer configuration settings</param>
- /// <param name="loggerFactory">Logger for creating loggers</param>
- public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory loggerFactory)
- {
- _client = client;
- _config = config;
- _logger = loggerFactory.CreateLogger<IggyConsumer>();
-
- _channel = Channel.CreateUnbounded<ReceivedMessage>();
- _consumerErrorEvents = new EventAggregator<ConsumerErrorEventArgs>(loggerFactory);
- }
-
- /// <summary>
- /// Disposes the consumer, leaving consumer groups and logging out if applicable
- /// </summary>
- public async ValueTask DisposeAsync()
- {
- if (Interlocked.Exchange(ref _disposeState, 1) == 1)
- {
- return;
- }
-
- if (_isInitialized)
- {
- _client.UnsubscribeConnectionEvents(OnClientConnectionStateChangedAsync);
- }
-
- if (!string.IsNullOrEmpty(_consumerGroupName) && _isInitialized)
- {
- try
- {
- await _client.LeaveConsumerGroupAsync(_config.StreamId, _config.TopicId,
- Identifier.String(_consumerGroupName));
-
- LogLeftConsumerGroup(_consumerGroupName);
- }
- catch (Exception e)
- {
- LogFailedToLeaveConsumerGroup(e, _consumerGroupName);
- }
- }
-
- if (_config.CreateIggyClient && _isInitialized)
- {
- try
- {
- await _client.LogoutUserAsync();
- _client.Dispose();
- }
- catch (Exception e)
- {
- LogFailedToLogoutOrDispose(e);
- }
- }
-
- _consumerErrorEvents.Clear();
- _pollingSemaphore.Dispose();
- _connectionStateSemaphore.Dispose();
- }
-
- /// <summary>
- /// Initializes the consumer by logging in (if needed) and setting up consumer groups
- /// </summary>
- /// <param name="ct">Cancellation token</param>
- /// <exception cref="InvalidConsumerGroupNameException">Thrown when consumer group name is invalid</exception>
- /// <exception cref="ConsumerGroupNotFoundException">Thrown when consumer group doesn't exist and auto-creation is disabled</exception>
- public async Task InitAsync(CancellationToken ct = default)
- {
- if (_isInitialized)
- {
- return;
- }
-
- await _connectionStateSemaphore.WaitAsync(ct);
- try
- {
- if (_isInitialized)
- {
- return;
- }
-
- if (_config.Consumer.Type == ConsumerType.ConsumerGroup && _config.PartitionId != null)
- {
- LogPartitionIdIsIgnoredWhenConsumerTypeIsConsumerGroup();
- _config.PartitionId = null;
- }
-
- await _client.ConnectAsync(ct);
-
- if (_config.CreateIggyClient)
- {
- await _client.LoginUserAsync(_config.Login, _config.Password, ct);
- }
-
- await InitializeConsumerGroupAsync(ct);
-
- _client.SubscribeConnectionEvents(OnClientConnectionStateChangedAsync);
-
- _isInitialized = true;
- }
- finally
- {
- _connectionStateSemaphore.Release();
- }
- }
-
- /// <summary>
- /// Receives messages asynchronously from the consumer as an async stream.
- /// Messages are automatically polled from the server and buffered in a bounded channel.
- /// </summary>
- /// <param name="ct">Cancellation token to stop receiving messages</param>
- /// <returns>An async enumerable of received messages</returns>
- /// <exception cref="ConsumerNotInitializedException">Thrown when InitAsync has not been called</exception>
- public async IAsyncEnumerable<ReceivedMessage> ReceiveAsync([EnumeratorCancellation] CancellationToken ct = default)
- {
- if (!_isInitialized)
- {
- throw new ConsumerNotInitializedException();
- }
-
- do
- {
- if (!_channel.Reader.TryRead(out var message))
- {
- await PollMessagesAsync(ct);
- continue;
- }
-
- yield return message;
-
- if (_config.AutoCommitMode == AutoCommitMode.AfterReceive)
- {
- await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct);
- }
- } while (!ct.IsCancellationRequested);
- }
-
- /// <summary>
- /// Manually stores the consumer offset for a specific partition.
- /// Use this when auto-commit is disabled or when you need manual offset control.
- /// </summary>
- /// <param name="offset">The offset to store</param>
- /// <param name="partitionId">The partition ID</param>
- /// <param name="resetLastPooled"></param>
- /// <param name="ct">Cancellation token</param>
- public async Task StoreOffsetAsync(ulong offset, uint partitionId, bool resetLastPooled = false,
- CancellationToken ct = default)
- {
- await _client.StoreOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, offset, partitionId, ct);
-
- if (resetLastPooled)
- {
- _lastPolledOffset[(int)partitionId] = offset;
- }
- }
-
- /// <summary>
- /// Deletes the stored consumer offset for a specific partition.
- /// The next poll will start from the beginning or based on the polling strategy.
- /// </summary>
- /// <param name="partitionId">The partition ID</param>
- /// <param name="ct">Cancellation token</param>
- public async Task DeleteOffsetAsync(uint partitionId, CancellationToken ct = default)
- {
- await _client.DeleteOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, partitionId, ct);
- }
-
- /// <summary>
- /// Event raised when an error occurs during polling.
- /// </summary>
- /// <param name="callback">Callback method</param>
- public void SubscribeToErrorEvents(Func<ConsumerErrorEventArgs, Task> callback)
- {
- _consumerErrorEvents.Subscribe(callback);
- }
-
- /// <summary>
- /// Unsubscribe from error events
- /// </summary>
- /// <param name="callback"></param>
- public void UnsubscribeFromErrorEvents(Func<ConsumerErrorEventArgs, Task> callback)
- {
- _consumerErrorEvents.Unsubscribe(callback);
- }
-
- /// <summary>
- /// Initializes consumer group if configured, creating and joining as needed
- /// </summary>
- private async Task InitializeConsumerGroupAsync(CancellationToken ct = default)
- {
- if (_joinedConsumerGroup)
- {
- return;
- }
-
- if (_config.Consumer.Type == ConsumerType.Consumer)
- {
- _joinedConsumerGroup = true;
- return;
- }
-
- _consumerGroupName = _config.Consumer.ConsumerId.Kind == IdKind.String
- ? _config.Consumer.ConsumerId.GetString()
- : _config.ConsumerGroupName;
-
- if (string.IsNullOrEmpty(_consumerGroupName))
- {
- throw new InvalidConsumerGroupNameException("Consumer group name is empty or null.");
- }
-
- try
- {
- var existingGroup = await _client.GetConsumerGroupByIdAsync(_config.StreamId, _config.TopicId,
- Identifier.String(_consumerGroupName), ct);
-
- if (existingGroup == null && _config.CreateConsumerGroupIfNotExists)
- {
- LogCreatingConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId);
-
- var createdGroup = await TryCreateConsumerGroupAsync(_consumerGroupName, ct);
-
- if (createdGroup)
- {
- LogConsumerGroupCreated(_consumerGroupName);
- }
- }
- else if (existingGroup == null)
- {
- throw new ConsumerGroupNotFoundException(_consumerGroupName);
- }
-
- if (_config.JoinConsumerGroup)
- {
- LogJoiningConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId);
-
- await _client.JoinConsumerGroupAsync(_config.StreamId, _config.TopicId,
- Identifier.String(_consumerGroupName), ct);
-
- _joinedConsumerGroup = true;
- LogConsumerGroupJoined(_consumerGroupName);
- }
- }
- catch (Exception ex)
- {
- LogFailedToInitializeConsumerGroup(ex, _consumerGroupName);
- throw;
- }
- }
-
- /// <summary>
- /// Attempts to create a consumer group, handling the case where it already exists
- /// </summary>
- /// <returns>True if the group was created or already exists, false on error</returns>
- private async Task<bool> TryCreateConsumerGroupAsync(string groupName, CancellationToken ct)
- {
- try
- {
- await _client.CreateConsumerGroupAsync(_config.StreamId, _config.TopicId,
- groupName, ct);
- }
- catch (IggyInvalidStatusCodeException ex)
- {
- // 5004 - Consumer group already exists TODO: refactor errors
- if (ex.StatusCode != 5004)
- {
- LogFailedToCreateConsumerGroup(ex, groupName);
- return false;
- }
-
- return true;
- }
-
- return true;
- }
-
- /// <summary>
- /// Polls messages from the server and writes them to the internal channel.
- /// Handles decryption, offset tracking, and auto-commit logic.
- /// Uses semaphore to ensure single concurrent polling operation.
- /// </summary>
- private async Task PollMessagesAsync(CancellationToken ct)
- {
- if (!_joinedConsumerGroup)
- {
- LogConsumerGroupNotJoinedYetSkippingPolling();
- return;
- }
-
- await _pollingSemaphore.WaitAsync(ct);
-
- try
- {
- if (_config.PollingIntervalMs > 0)
- {
- await WaitBeforePollingAsync(ct);
- }
-
- var messages = await _client.PollMessagesAsync(_config.StreamId, _config.TopicId,
- _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize,
- _config.AutoCommit, ct);
-
- var receiveMessages = messages.Messages.Count > 0;
-
- if (_lastPolledOffset.TryGetValue(messages.PartitionId, out var lastPolledPartitionOffset))
- {
- messages.Messages = messages.Messages.Where(x => x.Header.Offset > lastPolledPartitionOffset).ToList();
- }
-
- if (messages.Messages.Count == 0
- && receiveMessages
- && _config.AutoCommitMode != AutoCommitMode.Disabled)
- {
- _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}",
- lastPolledPartitionOffset, messages.PartitionId);
- await StoreOffsetAsync(lastPolledPartitionOffset, (uint)messages.PartitionId, false, ct);
- }
-
- if (messages.Messages.Count == 0)
- {
- return;
- }
-
- var currentOffset = 0ul;
- foreach (var message in messages.Messages)
- {
- var processedMessage = message;
- var status = MessageStatus.Success;
- Exception? error = null;
-
- if (_config.MessageEncryptor != null)
- {
- try
- {
- var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload);
-
- Dictionary<HeaderKey, HeaderValue>? decryptedHeaders = null;
- if (message.RawUserHeaders is { Length: > 0 })
- {
- var decryptedHeaderBytes = _config.MessageEncryptor.Decrypt(message.RawUserHeaders);
- decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes);
- }
-
- processedMessage = new MessageResponse
- {
- Header = message.Header,
- Payload = decryptedPayload,
- UserHeaders = decryptedHeaders
- };
- }
- catch (Exception ex)
- {
- LogFailedToDecryptMessage(ex, message.Header.Offset);
- status = MessageStatus.DecryptionFailed;
- error = ex;
- }
- }
-
- var receivedMessage = new ReceivedMessage
- {
- Message = processedMessage,
- CurrentOffset = processedMessage.Header.Offset,
- PartitionId = (uint)messages.PartitionId,
- Status = status,
- Error = error
- };
-
- await _channel.Writer.WriteAsync(receivedMessage, ct);
- currentOffset = receivedMessage.CurrentOffset;
- }
-
- _lastPolledOffset.AddOrUpdate(messages.PartitionId, currentOffset,
- (_, _) => currentOffset);
-
- if (_config.PollingStrategy.Kind == MessagePolling.Offset)
- {
- _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1);
- }
- }
- catch (Exception ex)
- {
- LogFailedToPollMessages(ex);
- _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages"));
- }
- finally
- {
- _pollingSemaphore.Release();
- }
- }
-
- /// <summary>
- /// Implements polling interval throttling to avoid excessive server requests.
- /// Uses monotonic time tracking to ensure proper intervals even with clock adjustments.
- /// </summary>
- private async Task WaitBeforePollingAsync(CancellationToken ct)
- {
- var intervalMs = _config.PollingIntervalMs;
- if (intervalMs <= 0)
- {
- return;
- }
-
- var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
- var lastPolledAtMs = Interlocked.Read(ref _lastPolledAtMs);
-
- if (nowMs < lastPolledAtMs)
- {
- LogMonotonicTimeWentBackwards(nowMs, lastPolledAtMs);
- await Task.Delay(intervalMs, ct);
- Interlocked.Exchange(ref _lastPolledAtMs, nowMs);
- return;
- }
-
- var elapsedMs = nowMs - lastPolledAtMs;
- if (elapsedMs >= intervalMs)
- {
- LogNoNeedToWaitBeforePolling(nowMs, lastPolledAtMs, elapsedMs);
- Interlocked.Exchange(ref _lastPolledAtMs, nowMs);
- return;
- }
-
- var remainingMs = intervalMs - elapsedMs;
- LogWaitingBeforePolling(remainingMs);
-
- if (remainingMs > 0)
- {
- await Task.Delay((int)remainingMs, ct);
- }
-
- Interlocked.Exchange(ref _lastPolledAtMs, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
- }
-
- /// <summary>
- /// Handles connection state changes from the client.
- /// </summary>
- /// <param name="e">Event object</param>
- private async Task OnClientConnectionStateChangedAsync(ConnectionStateChangedEventArgs e)
- {
- LogConnectionStateChanged(e.PreviousState, e.CurrentState);
-
- await _connectionStateSemaphore.WaitAsync();
- try
- {
- if (e.CurrentState == ConnectionState.Disconnected)
- {
- _joinedConsumerGroup = false;
- }
-
- if (e.CurrentState != ConnectionState.Authenticated
- || e.PreviousState == ConnectionState.Authenticated
- || _joinedConsumerGroup)
- {
- return;
- }
-
- await RejoinConsumerGroupOnReconnectionAsync();
- }
- finally
- {
- _connectionStateSemaphore.Release();
- }
- }
-
- /// <summary>
- /// Asynchronously rejoins the consumer group after a client reconnection.
- /// This restores the consumer group membership that was lost during the connection failure.
- /// </summary>
- private async Task RejoinConsumerGroupOnReconnectionAsync()
- {
- if (string.IsNullOrEmpty(_consumerGroupName))
- {
- LogConsumerGroupNameIsEmptySkippingRejoiningConsumerGroup();
- return;
- }
-
- try
- {
- await InitializeConsumerGroupAsync();
- }
- catch (Exception ex)
- {
- LogFailedToRejoinConsumerGroup(ex, _consumerGroupName);
- _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex,
- "Failed to rejoin consumer group after reconnection"));
- }
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Collections.Concurrent;
+using System.Runtime.CompilerServices;
+using System.Threading.Channels;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Headers;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Mappers;
+using Apache.Iggy.Utils;
+using Microsoft.Extensions.Logging;
+
+namespace Apache.Iggy.Consumers;
+
+/// <summary>
+/// High-level consumer for receiving messages from Iggy streams.
+/// Provides automatic polling, offset management, and consumer group support.
+/// </summary>
+public partial class IggyConsumer : IAsyncDisposable
+{
+ private readonly Channel<ReceivedMessage> _channel;
+ private readonly IIggyClient _client;
+ private readonly IggyConsumerConfig _config;
+ private readonly SemaphoreSlim _connectionStateSemaphore = new(1, 1);
+ private readonly EventAggregator<ConsumerErrorEventArgs> _consumerErrorEvents;
+ private readonly ConcurrentDictionary<int, ulong> _lastPolledOffset = new();
+ private readonly ILogger<IggyConsumer> _logger;
+ private readonly SemaphoreSlim _pollingSemaphore = new(1, 1);
+ private string? _consumerGroupName;
+ private int _disposeState;
+ private volatile bool _isInitialized;
+ private volatile bool _joinedConsumerGroup;
+ private long _lastPolledAtMs;
+
+ /// <summary>Whether this consumer has been initialized via <see cref="InitAsync" />.</summary>
+ protected bool IsInitialized => _isInitialized;
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="IggyConsumer" /> class
+ /// </summary>
+ /// <param name="client">The Iggy client for server communication</param>
+ /// <param name="config">Consumer configuration settings</param>
+ /// <param name="loggerFactory">Logger for creating loggers</param>
+ public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory loggerFactory)
+ {
+ _client = client;
+ _config = config;
+ _logger = loggerFactory.CreateLogger<IggyConsumer>();
+
+ _channel = Channel.CreateUnbounded<ReceivedMessage>();
+ _consumerErrorEvents = new EventAggregator<ConsumerErrorEventArgs>(loggerFactory);
+ }
+
+ /// <summary>
+ /// Disposes the consumer, leaving consumer groups and logging out if applicable
+ /// </summary>
+ public async ValueTask DisposeAsync()
+ {
+ if (Interlocked.Exchange(ref _disposeState, 1) == 1)
+ {
+ return;
+ }
+
+ if (_isInitialized)
+ {
+ _client.UnsubscribeConnectionEvents(OnClientConnectionStateChangedAsync);
+ }
+
+ if (!string.IsNullOrEmpty(_consumerGroupName) && _isInitialized)
+ {
+ try
+ {
+ await _client.LeaveConsumerGroupAsync(_config.StreamId, _config.TopicId,
+ Identifier.String(_consumerGroupName));
+
+ LogLeftConsumerGroup(_consumerGroupName);
+ }
+ catch (Exception e)
+ {
+ LogFailedToLeaveConsumerGroup(e, _consumerGroupName);
+ }
+ }
+
+ if (_config.CreateIggyClient && _isInitialized)
+ {
+ try
+ {
+ await _client.LogoutUserAsync();
+ _client.Dispose();
+ }
+ catch (Exception e)
+ {
+ LogFailedToLogoutOrDispose(e);
+ }
+ }
+
+ _consumerErrorEvents.Clear();
+ _pollingSemaphore.Dispose();
+ _connectionStateSemaphore.Dispose();
+ }
+
+ /// <summary>
+ /// Initializes the consumer by logging in (if needed) and setting up consumer groups
+ /// </summary>
+ /// <param name="ct">Cancellation token</param>
+ /// <exception cref="InvalidConsumerGroupNameException">Thrown when consumer group name is invalid</exception>
+ /// <exception cref="ConsumerGroupNotFoundException">Thrown when consumer group doesn't exist and auto-creation is disabled</exception>
+ public async Task InitAsync(CancellationToken ct = default)
+ {
+ if (_isInitialized)
+ {
+ return;
+ }
+
+ await _connectionStateSemaphore.WaitAsync(ct);
+ try
+ {
+ if (_isInitialized)
+ {
+ return;
+ }
+
+ if (_config.Consumer.Type == ConsumerType.ConsumerGroup && _config.PartitionId != null)
+ {
+ LogPartitionIdIsIgnoredWhenConsumerTypeIsConsumerGroup();
+ _config.PartitionId = null;
+ }
+
+ await _client.ConnectAsync(ct);
+
+ if (_config.CreateIggyClient)
+ {
+ await _client.LoginUserAsync(_config.Login, _config.Password, ct);
+ }
+
+ await InitializeConsumerGroupAsync(ct);
+
+ _client.SubscribeConnectionEvents(OnClientConnectionStateChangedAsync);
+
+ _isInitialized = true;
+ }
+ finally
+ {
+ _connectionStateSemaphore.Release();
+ }
+ }
+
+ /// <summary>
+ /// Receives messages asynchronously from the consumer as an async stream.
+ /// Messages are automatically polled from the server and buffered in a bounded channel.
+ /// </summary>
+ /// <param name="ct">Cancellation token to stop receiving messages</param>
+ /// <returns>An async enumerable of received messages</returns>
+ /// <exception cref="ConsumerNotInitializedException">Thrown when InitAsync has not been called</exception>
+ public async IAsyncEnumerable<ReceivedMessage> ReceiveAsync([EnumeratorCancellation] CancellationToken ct = default)
+ {
+ if (!_isInitialized)
+ {
+ throw new ConsumerNotInitializedException();
+ }
+
+ do
+ {
+ if (!_channel.Reader.TryRead(out var message))
+ {
+ await PollMessagesAsync(ct);
+ continue;
+ }
+
+ yield return message;
+
+ if (_config.AutoCommitMode == AutoCommitMode.AfterReceive)
+ {
+ await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct);
+ }
+ } while (!ct.IsCancellationRequested);
+ }
+
+ /// <summary>
+ /// Manually stores the consumer offset for a specific partition.
+ /// Use this when auto-commit is disabled or when you need manual offset control.
+ /// </summary>
+ /// <param name="offset">The offset to store</param>
+ /// <param name="partitionId">The partition ID</param>
+ /// <param name="resetLastPooled"></param>
+ /// <param name="ct">Cancellation token</param>
+ public async Task StoreOffsetAsync(ulong offset, uint partitionId, bool resetLastPooled = false,
+ CancellationToken ct = default)
+ {
+ await _client.StoreOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, offset, partitionId, ct);
+
+ if (resetLastPooled)
+ {
+ _lastPolledOffset[(int)partitionId] = offset;
+ }
+ }
+
+ /// <summary>
+ /// Deletes the stored consumer offset for a specific partition.
+ /// The next poll will start from the beginning or based on the polling strategy.
+ /// </summary>
+ /// <param name="partitionId">The partition ID</param>
+ /// <param name="ct">Cancellation token</param>
+ public async Task DeleteOffsetAsync(uint partitionId, CancellationToken ct = default)
+ {
+ await _client.DeleteOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, partitionId, ct);
+ }
+
+ /// <summary>
+ /// Event raised when an error occurs during polling.
+ /// </summary>
+ /// <param name="callback">Callback method</param>
+ public void SubscribeToErrorEvents(Func<ConsumerErrorEventArgs, Task> callback)
+ {
+ _consumerErrorEvents.Subscribe(callback);
+ }
+
+ /// <summary>
+ /// Unsubscribe from error events
+ /// </summary>
+ /// <param name="callback"></param>
+ public void UnsubscribeFromErrorEvents(Func<ConsumerErrorEventArgs, Task> callback)
+ {
+ _consumerErrorEvents.Unsubscribe(callback);
+ }
+
+ /// <summary>
+ /// Initializes consumer group if configured, creating and joining as needed
+ /// </summary>
+ private async Task InitializeConsumerGroupAsync(CancellationToken ct = default)
+ {
+ if (_joinedConsumerGroup)
+ {
+ return;
+ }
+
+ if (_config.Consumer.Type == ConsumerType.Consumer)
+ {
+ _joinedConsumerGroup = true;
+ return;
+ }
+
+ _consumerGroupName = _config.Consumer.ConsumerId.Kind == IdKind.String
+ ? _config.Consumer.ConsumerId.GetString()
+ : _config.ConsumerGroupName;
+
+ if (string.IsNullOrEmpty(_consumerGroupName))
+ {
+ throw new InvalidConsumerGroupNameException("Consumer group name is empty or null.");
+ }
+
+ try
+ {
+ var existingGroup = await _client.GetConsumerGroupByIdAsync(_config.StreamId, _config.TopicId,
+ Identifier.String(_consumerGroupName), ct);
+
+ if (existingGroup == null && _config.CreateConsumerGroupIfNotExists)
+ {
+ LogCreatingConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId);
+
+ var createdGroup = await TryCreateConsumerGroupAsync(_consumerGroupName, ct);
+
+ if (createdGroup)
+ {
+ LogConsumerGroupCreated(_consumerGroupName);
+ }
+ }
+ else if (existingGroup == null)
+ {
+ throw new ConsumerGroupNotFoundException(_consumerGroupName);
+ }
+
+ if (_config.JoinConsumerGroup)
+ {
+ LogJoiningConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId);
+
+ await _client.JoinConsumerGroupAsync(_config.StreamId, _config.TopicId,
+ Identifier.String(_consumerGroupName), ct);
+
+ _joinedConsumerGroup = true;
+ LogConsumerGroupJoined(_consumerGroupName);
+ }
+ }
+ catch (Exception ex)
+ {
+ LogFailedToInitializeConsumerGroup(ex, _consumerGroupName);
+ throw;
+ }
+ }
+
+ /// <summary>
+ /// Attempts to create a consumer group, handling the case where it already exists
+ /// </summary>
+ /// <returns>True if the group was created or already exists, false on error</returns>
+ private async Task<bool> TryCreateConsumerGroupAsync(string groupName, CancellationToken ct)
+ {
+ try
+ {
+ await _client.CreateConsumerGroupAsync(_config.StreamId, _config.TopicId,
+ groupName, ct);
+ }
+ catch (IggyInvalidStatusCodeException ex)
+ {
+ // 5004 - Consumer group already exists TODO: refactor errors
+ if (ex.StatusCode != 5004)
+ {
+ LogFailedToCreateConsumerGroup(ex, groupName);
+ return false;
+ }
+
+ return true;
+ }
+
+ return true;
+ }
+
+ /// <summary>
+ /// Polls messages from the server and writes them to the internal channel.
+ /// Handles decryption, offset tracking, and auto-commit logic.
+ /// Uses semaphore to ensure single concurrent polling operation.
+ /// </summary>
+ private async Task PollMessagesAsync(CancellationToken ct)
+ {
+ if (!_joinedConsumerGroup)
+ {
+ LogConsumerGroupNotJoinedYetSkippingPolling();
+ return;
+ }
+
+ await _pollingSemaphore.WaitAsync(ct);
+
+ try
+ {
+ if (_config.PollingIntervalMs > 0)
+ {
+ await WaitBeforePollingAsync(ct);
+ }
+
+ var messages = await _client.PollMessagesAsync(_config.StreamId, _config.TopicId,
+ _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize,
+ _config.AutoCommit, ct);
+
+ var receiveMessages = messages.Messages.Count > 0;
+
+ if (_lastPolledOffset.TryGetValue(messages.PartitionId, out var lastPolledPartitionOffset))
+ {
+ messages.Messages = messages.Messages.Where(x => x.Header.Offset > lastPolledPartitionOffset).ToList();
+ }
+
+ if (messages.Messages.Count == 0
+ && receiveMessages
+ && _config.AutoCommitMode != AutoCommitMode.Disabled)
+ {
+ _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}",
+ lastPolledPartitionOffset, messages.PartitionId);
+ await StoreOffsetAsync(lastPolledPartitionOffset, (uint)messages.PartitionId, false, ct);
+ }
+
+ if (messages.Messages.Count == 0)
+ {
+ return;
+ }
+
+ var currentOffset = 0ul;
+ foreach (var message in messages.Messages)
+ {
+ var processedMessage = message;
+ var status = MessageStatus.Success;
+ Exception? error = null;
+
+ if (_config.MessageEncryptor != null)
+ {
+ try
+ {
+ var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload);
+
+ Dictionary<HeaderKey, HeaderValue>? decryptedHeaders = null;
+ if (message.RawUserHeaders is { Length: > 0 })
+ {
+ var decryptedHeaderBytes = _config.MessageEncryptor.Decrypt(message.RawUserHeaders);
+ decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes);
+ }
+
+ processedMessage = new MessageResponse
+ {
+ Header = message.Header,
+ Payload = decryptedPayload,
+ UserHeaders = decryptedHeaders
+ };
+ }
+ catch (Exception ex)
+ {
+ LogFailedToDecryptMessage(ex, message.Header.Offset);
+ status = MessageStatus.DecryptionFailed;
+ error = ex;
+ }
+ }
+
+ var receivedMessage = new ReceivedMessage
+ {
+ Message = processedMessage,
+ CurrentOffset = processedMessage.Header.Offset,
+ PartitionId = (uint)messages.PartitionId,
+ Status = status,
+ Error = error
+ };
+
+ await _channel.Writer.WriteAsync(receivedMessage, ct);
+ currentOffset = receivedMessage.CurrentOffset;
+ }
+
+ _lastPolledOffset.AddOrUpdate(messages.PartitionId, currentOffset,
+ (_, _) => currentOffset);
+
+ if (_config.PollingStrategy.Kind == MessagePolling.Offset)
+ {
+ _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1);
+ }
+ }
+ catch (Exception ex)
+ {
+ LogFailedToPollMessages(ex);
+ _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages"));
+ }
+ finally
+ {
+ _pollingSemaphore.Release();
+ }
+ }
+
+ /// <summary>
+ /// Implements polling interval throttling to avoid excessive server requests.
+ /// Uses monotonic time tracking to ensure proper intervals even with clock adjustments.
+ /// </summary>
+ private async Task WaitBeforePollingAsync(CancellationToken ct)
+ {
+ var intervalMs = _config.PollingIntervalMs;
+ if (intervalMs <= 0)
+ {
+ return;
+ }
+
+ var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+ var lastPolledAtMs = Interlocked.Read(ref _lastPolledAtMs);
+
+ if (nowMs < lastPolledAtMs)
+ {
+ LogMonotonicTimeWentBackwards(nowMs, lastPolledAtMs);
+ await Task.Delay(intervalMs, ct);
+ Interlocked.Exchange(ref _lastPolledAtMs, nowMs);
+ return;
+ }
+
+ var elapsedMs = nowMs - lastPolledAtMs;
+ if (elapsedMs >= intervalMs)
+ {
+ LogNoNeedToWaitBeforePolling(nowMs, lastPolledAtMs, elapsedMs);
+ Interlocked.Exchange(ref _lastPolledAtMs, nowMs);
+ return;
+ }
+
+ var remainingMs = intervalMs - elapsedMs;
+ LogWaitingBeforePolling(remainingMs);
+
+ if (remainingMs > 0)
+ {
+ await Task.Delay((int)remainingMs, ct);
+ }
+
+ Interlocked.Exchange(ref _lastPolledAtMs, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
+ }
+
+ /// <summary>
+ /// Handles connection state changes from the client.
+ /// </summary>
+ /// <param name="e">Event object</param>
+ private async Task OnClientConnectionStateChangedAsync(ConnectionStateChangedEventArgs e)
+ {
+ LogConnectionStateChanged(e.PreviousState, e.CurrentState);
+
+ await _connectionStateSemaphore.WaitAsync();
+ try
+ {
+ if (e.CurrentState == ConnectionState.Disconnected)
+ {
+ _joinedConsumerGroup = false;
+ }
+
+ if (e.CurrentState != ConnectionState.Authenticated
+ || e.PreviousState == ConnectionState.Authenticated
+ || _joinedConsumerGroup)
+ {
+ return;
+ }
+
+ await RejoinConsumerGroupOnReconnectionAsync();
+ }
+ finally
+ {
+ _connectionStateSemaphore.Release();
+ }
+ }
+
+ /// <summary>
+ /// Asynchronously rejoins the consumer group after a client reconnection.
+ /// This restores the consumer group membership that was lost during the connection failure.
+ /// </summary>
+ private async Task RejoinConsumerGroupOnReconnectionAsync()
+ {
+ if (string.IsNullOrEmpty(_consumerGroupName))
+ {
+ LogConsumerGroupNameIsEmptySkippingRejoiningConsumerGroup();
+ return;
+ }
+
+ try
+ {
+ await InitializeConsumerGroupAsync();
+ }
+ catch (Exception ex)
+ {
+ LogFailedToRejoinConsumerGroup(ex, _consumerGroupName);
+ _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex,
+ "Failed to rejoin consumer group after reconnection"));
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs
index b8ea65b62..ecca568aa 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs
@@ -1,193 +1,193 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Runtime.CompilerServices;
-using System.Threading.Channels;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.IggyClient;
-using Microsoft.Extensions.Logging;
-
-namespace Apache.Iggy.Consumers;
-
-/// <summary>
-/// Typed consumer that automatically deserializes message payloads to type T.
-/// Extends <see cref="IggyConsumer" /> with deserialization capabilities.
-/// </summary>
-/// <typeparam name="T">The type to deserialize message payloads to</typeparam>
-public class IggyConsumer<T> : IggyConsumer
-{
- private readonly Channel<DeserializedMessage<T>> _deserializedChannel =
- Channel.CreateUnbounded<DeserializedMessage<T>>();
-
- private readonly IggyConsumerConfig<T> _typedConfig;
- private readonly ILogger<IggyConsumer<T>> _typedLogger;
-
- /// <summary>
- /// Initializes a new instance of the typed <see cref="IggyConsumer{T}" /> class
- /// </summary>
- /// <param name="client">The Iggy client for server communication</param>
- /// <param name="config">Typed consumer configuration including deserializer</param>
- /// <param name="logger">Logger instance for diagnostic output</param>
- public IggyConsumer(IIggyClient client, IggyConsumerConfig<T> config, ILoggerFactory logger) : base(client, config,
- logger)
- {
- _typedConfig = config;
- _typedLogger = logger.CreateLogger<IggyConsumer<T>>();
- }
-
- /// <summary>
- /// Receives and deserializes messages from the consumer
- /// </summary>
- /// <param name="ct">Cancellation token</param>
- /// <returns>Async enumerable of deserialized messages with status</returns>
- public async IAsyncEnumerable<ReceivedMessage<T>> ReceiveDeserializedAsync(
- [EnumeratorCancellation] CancellationToken ct = default)
- {
- await foreach (var message in ReceiveAsync(ct))
- {
- if (message.Status != MessageStatus.Success)
- {
- yield return new ReceivedMessage<T>
- {
- Data = default,
- Message = message.Message,
- CurrentOffset = message.CurrentOffset,
- PartitionId = message.PartitionId,
- Status = message.Status,
- Error = message.Error
- };
- continue;
- }
-
- T? deserializedPayload = default;
- Exception? deserializationError = null;
- var status = MessageStatus.Success;
-
- try
- {
- deserializedPayload = Deserialize(message.Message.Payload);
- }
- catch (Exception ex)
- {
- _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}", message.CurrentOffset);
- status = MessageStatus.DeserializationFailed;
- deserializationError = ex;
- }
-
- yield return new ReceivedMessage<T>
- {
- Data = deserializedPayload,
- Message = message.Message,
- CurrentOffset = message.CurrentOffset,
- PartitionId = message.PartitionId,
- Status = status,
- Error = deserializationError
- };
- }
- }
-
- /// <summary>
- /// Receives and deserializes messages via the rented poll path. Each polled batch is deserialized
- /// in full before any message is yielded — the rented buffer is returned immediately after
- /// deserialization, independently of how fast the caller iterates. The caller does not need to
- /// dispose anything.
- /// </summary>
- /// <param name="ct">Cancellation token.</param>
- /// <returns>Async enumerable of deserialized messages with status.</returns>
- public async IAsyncEnumerable<DeserializedMessage<T>> ReceiveRentedDeserializedAsync(
- [EnumeratorCancellation] CancellationToken ct = default)
- {
- if (!IsInitialized)
- {
- throw new ConsumerNotInitializedException();
- }
-
- do
- {
- if (!_deserializedChannel.Reader.TryRead(out DeserializedMessage<T>? message))
- {
- await PollRentedMessagesAsync(ct);
- continue;
- }
-
- yield return message;
-
- if (_typedConfig.AutoCommitMode == AutoCommitMode.AfterReceive)
- {
- await StoreOffsetAsync(message.Header.Offset, message.PartitionId, false, ct);
- }
- } while (!ct.IsCancellationRequested);
- }
-
- /// <summary>
- /// Overrides the base batch-publishing step: instead of routing rented messages through the base
- /// class channel, deserializes the entire batch immediately (releasing all rented buffer refs via
- /// <see cref="IDisposable.Dispose" />) and writes the deserialized results to
- /// <see cref="_deserializedChannel" />. Auto-commit is also handled here since the base-class
- /// yield path is bypassed.
- /// </summary>
- protected override async Task PublishRentedAsync(RentedBatchHandle rental, RentedMessageResponse message,
- uint partitionId, MessageStatus status,
- Exception? error, CancellationToken ct)
- {
- T? data = default;
- var deserError = status != MessageStatus.Success ? error : null;
- var msgStatus = status;
-
- if (status == MessageStatus.Success)
- {
- try
- {
- data = Deserialize(message.Payload);
- }
- catch (Exception ex)
- {
- _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}",
- message.Header.Offset);
- msgStatus = MessageStatus.DeserializationFailed;
- deserError = ex;
- }
- }
-
- var deserialized = new DeserializedMessage<T>
- {
- Data = data,
- Header = message.Header,
- UserHeaders = message.UserHeaders,
- CurrentOffset = message.Header.Offset,
- PartitionId = partitionId,
- Status = msgStatus,
- Error = deserError
- };
-
- await _deserializedChannel.Writer.WriteAsync(deserialized, ct);
-
- rental.Release();
- }
-
- /// <summary>
- /// Deserializes a message payload from a span using the configured deserializer. Zero-copy when the
- /// deserializer overrides the span overload; otherwise falls back to a one-time array copy.
- /// </summary>
- /// <param name="payload">The payload memory to deserialize.</param>
- /// <returns>The deserialized object of type T.</returns>
- public T Deserialize(ReadOnlyMemory<byte> payload)
- {
- return _typedConfig.Deserializer.Deserialize(payload);
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Runtime.CompilerServices;
+using System.Threading.Channels;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.IggyClient;
+using Microsoft.Extensions.Logging;
+
+namespace Apache.Iggy.Consumers;
+
+/// <summary>
+/// Typed consumer that automatically deserializes message payloads to type T.
+/// Extends <see cref="IggyConsumer" /> with deserialization capabilities.
+/// </summary>
+/// <typeparam name="T">The type to deserialize message payloads to</typeparam>
+public class IggyConsumer<T> : IggyConsumer
+{
+ private readonly Channel<DeserializedMessage<T>> _deserializedChannel =
+ Channel.CreateUnbounded<DeserializedMessage<T>>();
+
+ private readonly IggyConsumerConfig<T> _typedConfig;
+ private readonly ILogger<IggyConsumer<T>> _typedLogger;
+
+ /// <summary>
+ /// Initializes a new instance of the typed <see cref="IggyConsumer{T}" /> class
+ /// </summary>
+ /// <param name="client">The Iggy client for server communication</param>
+ /// <param name="config">Typed consumer configuration including deserializer</param>
+ /// <param name="logger">Logger instance for diagnostic output</param>
+ public IggyConsumer(IIggyClient client, IggyConsumerConfig<T> config, ILoggerFactory logger) : base(client, config,
+ logger)
+ {
+ _typedConfig = config;
+ _typedLogger = logger.CreateLogger<IggyConsumer<T>>();
+ }
+
+ /// <summary>
+ /// Receives and deserializes messages from the consumer
+ /// </summary>
+ /// <param name="ct">Cancellation token</param>
+ /// <returns>Async enumerable of deserialized messages with status</returns>
+ public async IAsyncEnumerable<ReceivedMessage<T>> ReceiveDeserializedAsync(
+ [EnumeratorCancellation] CancellationToken ct = default)
+ {
+ await foreach (var message in ReceiveAsync(ct))
+ {
+ if (message.Status != MessageStatus.Success)
+ {
+ yield return new ReceivedMessage<T>
+ {
+ Data = default,
+ Message = message.Message,
+ CurrentOffset = message.CurrentOffset,
+ PartitionId = message.PartitionId,
+ Status = message.Status,
+ Error = message.Error
+ };
+ continue;
+ }
+
+ T? deserializedPayload = default;
+ Exception? deserializationError = null;
+ var status = MessageStatus.Success;
+
+ try
+ {
+ deserializedPayload = Deserialize(message.Message.Payload);
+ }
+ catch (Exception ex)
+ {
+ _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}", message.CurrentOffset);
+ status = MessageStatus.DeserializationFailed;
+ deserializationError = ex;
+ }
+
+ yield return new ReceivedMessage<T>
+ {
+ Data = deserializedPayload,
+ Message = message.Message,
+ CurrentOffset = message.CurrentOffset,
+ PartitionId = message.PartitionId,
+ Status = status,
+ Error = deserializationError
+ };
+ }
+ }
+
+ /// <summary>
+ /// Receives and deserializes messages via the rented poll path. Each polled batch is deserialized
+ /// in full before any message is yielded — the rented buffer is returned immediately after
+ /// deserialization, independently of how fast the caller iterates. The caller does not need to
+ /// dispose anything.
+ /// </summary>
+ /// <param name="ct">Cancellation token.</param>
+ /// <returns>Async enumerable of deserialized messages with status.</returns>
+ public async IAsyncEnumerable<DeserializedMessage<T>> ReceiveRentedDeserializedAsync(
+ [EnumeratorCancellation] CancellationToken ct = default)
+ {
+ if (!IsInitialized)
+ {
+ throw new ConsumerNotInitializedException();
+ }
+
+ do
+ {
+ if (!_deserializedChannel.Reader.TryRead(out DeserializedMessage<T>? message))
+ {
+ await PollRentedMessagesAsync(ct);
+ continue;
+ }
+
+ yield return message;
+
+ if (_typedConfig.AutoCommitMode == AutoCommitMode.AfterReceive)
+ {
+ await StoreOffsetAsync(message.Header.Offset, message.PartitionId, false, ct);
+ }
+ } while (!ct.IsCancellationRequested);
+ }
+
+ /// <summary>
+ /// Overrides the base batch-publishing step: instead of routing rented messages through the base
+ /// class channel, deserializes the entire batch immediately (releasing all rented buffer refs via
+ /// <see cref="IDisposable.Dispose" />) and writes the deserialized results to
+ /// <see cref="_deserializedChannel" />. Auto-commit is also handled here since the base-class
+ /// yield path is bypassed.
+ /// </summary>
+ protected override async Task PublishRentedAsync(RentedBatchHandle rental, RentedMessageResponse message,
+ uint partitionId, MessageStatus status,
+ Exception? error, CancellationToken ct)
+ {
+ T? data = default;
+ var deserError = status != MessageStatus.Success ? error : null;
+ var msgStatus = status;
+
+ if (status == MessageStatus.Success)
+ {
+ try
+ {
+ data = Deserialize(message.Payload);
+ }
+ catch (Exception ex)
+ {
+ _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}",
+ message.Header.Offset);
+ msgStatus = MessageStatus.DeserializationFailed;
+ deserError = ex;
+ }
+ }
+
+ var deserialized = new DeserializedMessage<T>
+ {
+ Data = data,
+ Header = message.Header,
+ UserHeaders = message.UserHeaders,
+ CurrentOffset = message.Header.Offset,
+ PartitionId = partitionId,
+ Status = msgStatus,
+ Error = deserError
+ };
+
+ await _deserializedChannel.Writer.WriteAsync(deserialized, ct);
+
+ rental.Release();
+ }
+
+ /// <summary>
+ /// Deserializes a message payload from a span using the configured deserializer. Zero-copy when the
+ /// deserializer overrides the span overload; otherwise falls back to a one-time array copy.
+ /// </summary>
+ /// <param name="payload">The payload memory to deserialize.</param>
+ /// <returns>The deserialized object of type T.</returns>
+ public T Deserialize(ReadOnlyMemory<byte> payload)
+ {
+ return _typedConfig.Deserializer.Deserialize(payload);
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs
index ac0f92e6c..60ac5ac7b 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs
@@ -1,126 +1,126 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using Apache.Iggy.Contracts;
-
-namespace Apache.Iggy.Consumers;
-
-/// <summary>
-/// Represents a message received from the Iggy consumer whose payload and raw headers are backed by rented memory
-/// shared across a polled batch. The caller SHOULD dispose the message once processing is complete for deterministic
-/// release of the underlying pool buffer. If the caller forgets, the buffer is still returned to the pool when the
-/// owning <see cref="PolledMessagesRental" /> becomes unreachable and its finalizer runs - non-deterministic but safe.
-/// </summary>
-public sealed class ReceivedRentedMessage : IDisposable
-{
- private int _disposed;
- internal RentedBatchHandle? Handle { get; init; }
-
- /// <summary>
- /// The underlying rented message response containing headers and rented payload memory.
- /// </summary>
- public required RentedMessageResponse Message { get; init; }
-
- /// <summary>
- /// The current offset of this message in the partition.
- /// </summary>
- public required ulong CurrentOffset { get; init; }
-
- /// <summary>
- /// The partition ID from which this message was consumed.
- /// </summary>
- public uint PartitionId { get; init; }
-
- /// <summary>
- /// The status of the message (Success, DecryptionFailed).
- /// </summary>
- public MessageStatus Status { get; init; } = MessageStatus.Success;
-
- /// <summary>
- /// The exception that occurred during processing, if any.
- /// </summary>
- public Exception? Error { get; init; }
-
- /// <summary>
- /// Releases this message's reference on the underlying rental. When the final message of a batch is disposed,
- /// the rented buffer is returned to the pool and any payload/raw header slices are invalidated.
- /// </summary>
- public void Dispose()
- {
- if (Interlocked.Exchange(ref _disposed, 1) == 1)
- {
- return;
- }
-
- Handle?.Release();
- }
-}
-
-/// <summary>
-/// Reference-counted handle around a <see cref="PolledMessagesRental" />. Shared by all
-/// <see cref="ReceivedRentedMessage" /> instances produced from one poll; the rental is disposed
-/// when the reference count drops to zero.
-/// </summary>
-public sealed class RentedBatchHandle : IDisposable
-{
- private readonly PolledMessagesRental _rental;
- private int _refCount = 1;
-
- /// <summary>
- /// Creates a new handle with a single self-reference held by the constructing producer. The producer
- /// must call <see cref="Acquire" /> before each publish and release the self-reference (via
- /// <see cref="Dispose" /> or <see cref="Release" />) once it has finished producing.
- /// </summary>
- /// <param name="rental">The polled messages rental whose lifetime this handle manages.</param>
- public RentedBatchHandle(PolledMessagesRental rental)
- {
- _rental = rental;
- }
-
- /// <summary>
- /// Releases one reference on the underlying rental. Equivalent to <see cref="Release" />.
- /// </summary>
- public void Dispose()
- {
- Release();
- }
-
- /// <summary>
- /// Acquires an additional reference. Must be balanced by a matching <see cref="Release" />.
- /// </summary>
- public void Acquire()
- {
- Interlocked.Increment(ref _refCount);
- }
-
- /// <summary>
- /// Decrements the reference count. When the count reaches zero, the underlying rental is disposed
- /// and its pool buffer returned.
- /// </summary>
- public void Release()
- {
- var remaining = Interlocked.Decrement(ref _refCount);
- if (remaining == 0)
- {
- _rental.Dispose();
- }
- else if (remaining < 0)
- {
- throw new InvalidOperationException("RentedBatchHandle released more times than acquired.");
- }
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Contracts;
+
+namespace Apache.Iggy.Consumers;
+
+/// <summary>
+/// Represents a message received from the Iggy consumer whose payload and raw headers are backed by rented memory
+/// shared across a polled batch. The caller SHOULD dispose the message once processing is complete for deterministic
+/// release of the underlying pool buffer. If the caller forgets, the buffer is still returned to the pool when the
+/// owning <see cref="PolledMessagesRental" /> becomes unreachable and its finalizer runs - non-deterministic but safe.
+/// </summary>
+public sealed class ReceivedRentedMessage : IDisposable
+{
+ private int _disposed;
+ internal RentedBatchHandle? Handle { get; init; }
+
+ /// <summary>
+ /// The underlying rented message response containing headers and rented payload memory.
+ /// </summary>
+ public required RentedMessageResponse Message { get; init; }
+
+ /// <summary>
+ /// The current offset of this message in the partition.
+ /// </summary>
+ public required ulong CurrentOffset { get; init; }
+
+ /// <summary>
+ /// The partition ID from which this message was consumed.
+ /// </summary>
+ public uint PartitionId { get; init; }
+
+ /// <summary>
+ /// The status of the message (Success, DecryptionFailed).
+ /// </summary>
+ public MessageStatus Status { get; init; } = MessageStatus.Success;
+
+ /// <summary>
+ /// The exception that occurred during processing, if any.
+ /// </summary>
+ public Exception? Error { get; init; }
+
+ /// <summary>
+ /// Releases this message's reference on the underlying rental. When the final message of a batch is disposed,
+ /// the rented buffer is returned to the pool and any payload/raw header slices are invalidated.
+ /// </summary>
+ public void Dispose()
+ {
+ if (Interlocked.Exchange(ref _disposed, 1) == 1)
+ {
+ return;
+ }
+
+ Handle?.Release();
+ }
+}
+
+/// <summary>
+/// Reference-counted handle around a <see cref="PolledMessagesRental" />. Shared by all
+/// <see cref="ReceivedRentedMessage" /> instances produced from one poll; the rental is disposed
+/// when the reference count drops to zero.
+/// </summary>
+public sealed class RentedBatchHandle : IDisposable
+{
+ private readonly PolledMessagesRental _rental;
+ private int _refCount = 1;
+
+ /// <summary>
+ /// Creates a new handle with a single self-reference held by the constructing producer. The producer
+ /// must call <see cref="Acquire" /> before each publish and release the self-reference (via
+ /// <see cref="Dispose" /> or <see cref="Release" />) once it has finished producing.
+ /// </summary>
+ /// <param name="rental">The polled messages rental whose lifetime this handle manages.</param>
+ public RentedBatchHandle(PolledMessagesRental rental)
+ {
+ _rental = rental;
+ }
+
+ /// <summary>
+ /// Releases one reference on the underlying rental. Equivalent to <see cref="Release" />.
+ /// </summary>
+ public void Dispose()
+ {
+ Release();
+ }
+
+ /// <summary>
+ /// Acquires an additional reference. Must be balanced by a matching <see cref="Release" />.
+ /// </summary>
+ public void Acquire()
+ {
+ Interlocked.Increment(ref _refCount);
+ }
+
+ /// <summary>
+ /// Decrements the reference count. When the count reaches zero, the underlying rental is disposed
+ /// and its pool buffer returned.
+ /// </summary>
+ public void Release()
+ {
+ var remaining = Interlocked.Decrement(ref _refCount);
+ if (remaining == 0)
+ {
+ _rental.Dispose();
+ }
+ else if (remaining < 0)
+ {
+ throw new InvalidOperationException("RentedBatchHandle released more times than acquired.");
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs
index 7895e7921..f29ca79fa 100644
--- a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs
+++ b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs
@@ -1,96 +1,96 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Text.Json.Serialization;
-using Apache.Iggy.Headers;
-using Apache.Iggy.JsonConverters;
-using Apache.Iggy.Mappers;
-using Apache.Iggy.Messages;
-
-namespace Apache.Iggy.Contracts;
-
-/// <summary>
-/// Response from the server containing a message payload.
-/// </summary>
-[JsonConverter(typeof(MessageResponseConverter))]
-public sealed class MessageResponse
-{
- private byte[]? _rawUserHeaders;
- private Dictionary<HeaderKey, HeaderValue>? _userHeaders;
- private bool _userHeadersInitialized;
-
- /// <summary>
- /// Message header.
- /// </summary>
- public required MessageHeader Header { get; set; }
-
- /// <summary>
- /// Message payload.
- /// </summary>
- public required byte[] Payload { get; set; } = [];
-
- /// <summary>
- /// Headers defined by the user.
- /// </summary>
- public Dictionary<HeaderKey, HeaderValue>? UserHeaders
- {
- get
- {
- if (!_userHeadersInitialized)
- {
- _userHeaders = _rawUserHeaders is { Length: > 0 }
- ? BinaryMapper.TryMapHeaders(_rawUserHeaders)
- : null;
- _userHeadersInitialized = true;
- }
-
- return _userHeaders;
- }
- set
- {
- _userHeaders = value;
- _userHeadersInitialized = true;
- }
- }
-
- /// <summary>
- /// Raw user header bytes before deserialization.
- /// Used internally for decrypting encrypted headers.
- /// </summary>
- [JsonIgnore]
- internal byte[]? RawUserHeaders
- {
- get => _rawUserHeaders;
- set
- {
- _rawUserHeaders = value;
- _userHeaders = null;
- _userHeadersInitialized = false;
- }
- }
-
- internal void ParseUserHeaders()
- {
- if (!_userHeadersInitialized)
- {
- _userHeaders = _rawUserHeaders is { Length: > 0 }
- ? BinaryMapper.TryMapHeaders(_rawUserHeaders)
- : null;
- _userHeadersInitialized = true;
- }
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Text.Json.Serialization;
+using Apache.Iggy.Headers;
+using Apache.Iggy.JsonConverters;
+using Apache.Iggy.Mappers;
+using Apache.Iggy.Messages;
+
+namespace Apache.Iggy.Contracts;
+
+/// <summary>
+/// Response from the server containing a message payload.
+/// </summary>
+[JsonConverter(typeof(MessageResponseConverter))]
+public sealed class MessageResponse
+{
+ private byte[]? _rawUserHeaders;
+ private Dictionary<HeaderKey, HeaderValue>? _userHeaders;
+ private bool _userHeadersInitialized;
+
+ /// <summary>
+ /// Message header.
+ /// </summary>
+ public required MessageHeader Header { get; set; }
+
+ /// <summary>
+ /// Message payload.
+ /// </summary>
+ public required byte[] Payload { get; set; } = [];
+
+ /// <summary>
+ /// Headers defined by the user.
+ /// </summary>
+ public Dictionary<HeaderKey, HeaderValue>? UserHeaders
+ {
+ get
+ {
+ if (!_userHeadersInitialized)
+ {
+ _userHeaders = _rawUserHeaders is { Length: > 0 }
+ ? BinaryMapper.TryMapHeaders(_rawUserHeaders)
+ : null;
+ _userHeadersInitialized = true;
+ }
+
+ return _userHeaders;
+ }
+ set
+ {
+ _userHeaders = value;
+ _userHeadersInitialized = true;
+ }
+ }
+
+ /// <summary>
+ /// Raw user header bytes before deserialization.
+ /// Used internally for decrypting encrypted headers.
+ /// </summary>
+ [JsonIgnore]
+ internal byte[]? RawUserHeaders
+ {
+ get => _rawUserHeaders;
+ set
+ {
+ _rawUserHeaders = value;
+ _userHeaders = null;
+ _userHeadersInitialized = false;
+ }
+ }
+
+ internal void ParseUserHeaders()
+ {
+ if (!_userHeadersInitialized)
+ {
+ _userHeaders = _rawUserHeaders is { Length: > 0 }
+ ? BinaryMapper.TryMapHeaders(_rawUserHeaders)
+ : null;
+ _userHeadersInitialized = true;
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs b/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs
index 04a82834b..17a0ff654 100644
--- a/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs
+++ b/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs
@@ -1,71 +1,71 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Buffers;
-
-namespace Apache.Iggy.Contracts;
-
-/// <summary>
-/// Represents a rented poll result whose payload and raw header memory remain valid until disposed.
-/// </summary>
-public sealed class PolledMessagesRental : IDisposable
-{
- private readonly IMemoryOwner<byte> _owner;
- private int _disposed;
-
- /// <summary>
- /// Partition identifier for the messages.
- /// </summary>
- public required int PartitionId { get; init; }
-
- /// <summary>
- /// Current offset for the partition.
- /// </summary>
- public required ulong CurrentOffset { get; init; }
-
- /// <summary>
- /// Rented messages.
- /// </summary>
- public required IReadOnlyList<RentedMessageResponse> Messages { get; init; }
-
- internal PolledMessagesRental(IMemoryOwner<byte> owner)
- {
- _owner = owner;
- }
-
- /// <summary>
- /// Disposes the rental and returns the underlying buffer to the pool.
- /// </summary>
- public void Dispose()
- {
- if (Interlocked.Exchange(ref _disposed, 1) != 0)
- {
- return;
- }
-
- _owner.Dispose();
- GC.SuppressFinalize(this);
- }
-
- /// <summary>
- /// Finalizer fallback that returns the buffer to the pool if the caller forgot to dispose.
- /// </summary>
- ~PolledMessagesRental()
- {
- Dispose();
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Buffers;
+
+namespace Apache.Iggy.Contracts;
+
+/// <summary>
+/// Represents a rented poll result whose payload and raw header memory remain valid until disposed.
+/// </summary>
+public sealed class PolledMessagesRental : IDisposable
+{
+ private readonly IMemoryOwner<byte> _owner;
+ private int _disposed;
+
+ /// <summary>
+ /// Partition identifier for the messages.
+ /// </summary>
+ public required int PartitionId { get; init; }
+
+ /// <summary>
+ /// Current offset for the partition.
+ /// </summary>
+ public required ulong CurrentOffset { get; init; }
+
+ /// <summary>
+ /// Rented messages.
+ /// </summary>
+ public required IReadOnlyList<RentedMessageResponse> Messages { get; init; }
+
+ internal PolledMessagesRental(IMemoryOwner<byte> owner)
+ {
+ _owner = owner;
+ }
+
+ /// <summary>
+ /// Disposes the rental and returns the underlying buffer to the pool.
+ /// </summary>
+ public void Dispose()
+ {
+ if (Interlocked.Exchange(ref _disposed, 1) != 0)
+ {
+ return;
+ }
+
+ _owner.Dispose();
+ GC.SuppressFinalize(this);
+ }
+
+ /// <summary>
+ /// Finalizer fallback that returns the buffer to the pool if the caller forgot to dispose.
+ /// </summary>
+ ~PolledMessagesRental()
+ {
+ Dispose();
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs
index 43c629380..f4db2413a 100644
--- a/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs
+++ b/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs
@@ -1,71 +1,71 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using Apache.Iggy.Headers;
-using Apache.Iggy.Mappers;
-using Apache.Iggy.Messages;
-
-namespace Apache.Iggy.Contracts;
-
-/// <summary>
-/// Response containing rented payload and raw header memory.
-/// The payload and raw headers are only valid while the owning <see cref="PolledMessagesRental" /> is alive.
-/// </summary>
-public sealed class RentedMessageResponse
-{
- private Dictionary<HeaderKey, HeaderValue>? _userHeaders;
- private bool _userHeadersInitialized;
-
- /// <summary>
- /// Message header.
- /// </summary>
- public required MessageHeader Header { get; init; }
-
- /// <summary>
- /// Message payload backed by rented memory.
- /// </summary>
- public required ReadOnlyMemory<byte> Payload { get; init; }
-
- /// <summary>
- /// Raw user header bytes backed by rented memory.
- /// </summary>
- public ReadOnlyMemory<byte> RawUserHeaders { get; init; }
-
- /// <summary>
- /// Parsed user headers. Parsed lazily and cached on first access.
- /// </summary>
- public Dictionary<HeaderKey, HeaderValue>? UserHeaders
- {
- get
- {
- if (!_userHeadersInitialized)
- {
- _userHeaders = RawUserHeaders.IsEmpty
- ? null
- : BinaryMapper.TryMapHeaders(RawUserHeaders.Span);
- _userHeadersInitialized = true;
- }
-
- return _userHeaders;
- }
- init
- {
- _userHeaders = value;
- _userHeadersInitialized = true;
- }
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Headers;
+using Apache.Iggy.Mappers;
+using Apache.Iggy.Messages;
+
+namespace Apache.Iggy.Contracts;
+
+/// <summary>
+/// Response containing rented payload and raw header memory.
+/// The payload and raw headers are only valid while the owning <see cref="PolledMessagesRental" /> is alive.
+/// </summary>
+public sealed class RentedMessageResponse
+{
+ private Dictionary<HeaderKey, HeaderValue>? _userHeaders;
+ private bool _userHeadersInitialized;
+
+ /// <summary>
+ /// Message header.
+ /// </summary>
+ public required MessageHeader Header { get; init; }
+
+ /// <summary>
+ /// Message payload backed by rented memory.
+ /// </summary>
+ public required ReadOnlyMemory<byte> Payload { get; init; }
+
+ /// <summary>
+ /// Raw user header bytes backed by rented memory.
+ /// </summary>
+ public ReadOnlyMemory<byte> RawUserHeaders { get; init; }
+
+ /// <summary>
+ /// Parsed user headers. Parsed lazily and cached on first access.
+ /// </summary>
+ public Dictionary<HeaderKey, HeaderValue>? UserHeaders
+ {
+ get
+ {
+ if (!_userHeadersInitialized)
+ {
+ _userHeaders = RawUserHeaders.IsEmpty
+ ? null
+ : BinaryMapper.TryMapHeaders(RawUserHeaders.Span);
+ _userHeadersInitialized = true;
+ }
+
+ return _userHeaders;
+ }
+ init
+ {
+ _userHeaders = value;
+ _userHeadersInitialized = true;
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs b/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs
index 6aaf4d822..9c99f501d 100644
--- a/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs
+++ b/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs
@@ -1,129 +1,129 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Security.Cryptography;
-
-namespace Apache.Iggy.Encryption;
-
-/// <summary>
-/// AES-256-GCM based message encryptor for secure message encryption/decryption.
-/// Uses AES-GCM (Galois/Counter Mode) which provides both confidentiality and authenticity.
-/// </summary>
-public sealed class AesMessageEncryptor : IMessageEncryptor
-{
- private const int NonceSize = 12; // 96 bits - recommended for GCM
- private const int TagSize = 16; // 128 bits authentication tag
- private readonly byte[] _key;
-
- /// <summary>
- /// Creates a new AES message encryptor with the specified key.
- /// </summary>
- /// <param name="key">The encryption key. Must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256 respectively.</param>
- /// <exception cref="ArgumentException">Thrown when key length is invalid</exception>
- public AesMessageEncryptor(byte[] key)
- {
- if (key.Length != 16 && key.Length != 24 && key.Length != 32)
- {
- throw new ArgumentException("Key must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256",
- nameof(key));
- }
-
- _key = key;
- }
-
- /// <summary>
- /// Encrypts the provided plain data using AES-GCM.
- /// Format: [12-byte nonce][encrypted data][16-byte authentication tag]
- /// </summary>
- /// <param name="plainData">The plain data to encrypt</param>
- /// <returns>The encrypted data with nonce and tag</returns>
- public byte[] Encrypt(Span<byte> plainData)
- {
- using var aesGcm = new AesGcm(_key, TagSize);
-
- var nonce = new byte[NonceSize];
- RandomNumberGenerator.Fill(nonce);
-
- var ciphertext = new byte[plainData.Length];
- var tag = new byte[TagSize];
-
- aesGcm.Encrypt(nonce, plainData, ciphertext, tag);
-
- // Combine: nonce + ciphertext + tag
- var result = new byte[NonceSize + ciphertext.Length + TagSize];
- Buffer.BlockCopy(nonce, 0, result, 0, NonceSize);
- Buffer.BlockCopy(ciphertext, 0, result, NonceSize, ciphertext.Length);
- Buffer.BlockCopy(tag, 0, result, NonceSize + ciphertext.Length, TagSize);
-
- return result;
- }
-
- /// <summary>
- /// Decrypts the provided encrypted data using AES-GCM.
- /// Expected format: [12-byte nonce][encrypted data][16-byte authentication tag]
- /// </summary>
- /// <param name="encryptedData">The encrypted data with nonce and tag</param>
- /// <returns>The decrypted plain data</returns>
- /// <exception cref="ArgumentException">Thrown when encrypted data format is invalid</exception>
- /// <exception cref="CryptographicException">Thrown when decryption or authentication fails</exception>
- public byte[] Decrypt(ReadOnlySpan<byte> encryptedData)
- {
- if (encryptedData.Length < NonceSize + TagSize)
- {
- throw new ArgumentException("Encrypted data is too short to contain nonce and tag", nameof(encryptedData));
- }
-
- using var aesGcm = new AesGcm(_key, TagSize);
-
- // Extract nonce, ciphertext, and tag
- Span<byte> nonce = stackalloc byte[NonceSize];
- Span<byte> tag = stackalloc byte[TagSize];
- var ciphertextLength = encryptedData.Length - NonceSize - TagSize;
- Span<byte> ciphertext = stackalloc byte[ciphertextLength];
-
- encryptedData[..NonceSize].CopyTo(nonce);
- encryptedData.Slice(NonceSize, ciphertextLength).CopyTo(ciphertext);
- encryptedData.Slice(NonceSize + ciphertextLength, TagSize).CopyTo(tag);
-
- var plaintext = new byte[ciphertextLength];
- aesGcm.Decrypt(nonce, ciphertext, tag, plaintext);
-
- return plaintext;
- }
-
- /// <summary>
- /// Creates a new AES-256 message encryptor with the specified base64-encoded key.
- /// </summary>
- /// <param name="base64Key">The base64-encoded encryption key</param>
- /// <returns>A new AesMessageEncryptor instance</returns>
- public static AesMessageEncryptor FromBase64Key(string base64Key)
- {
- return new AesMessageEncryptor(Convert.FromBase64String(base64Key));
- }
-
- /// <summary>
- /// Generates a new random AES-256 key.
- /// </summary>
- /// <returns>A 32-byte random key suitable for AES-256</returns>
- public static byte[] GenerateKey()
- {
- var key = new byte[32];
- using var rng = RandomNumberGenerator.Create();
- rng.GetBytes(key);
- return key;
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Security.Cryptography;
+
+namespace Apache.Iggy.Encryption;
+
+/// <summary>
+/// AES-256-GCM based message encryptor for secure message encryption/decryption.
+/// Uses AES-GCM (Galois/Counter Mode) which provides both confidentiality and authenticity.
+/// </summary>
+public sealed class AesMessageEncryptor : IMessageEncryptor
+{
+ private const int NonceSize = 12; // 96 bits - recommended for GCM
+ private const int TagSize = 16; // 128 bits authentication tag
+ private readonly byte[] _key;
+
+ /// <summary>
+ /// Creates a new AES message encryptor with the specified key.
+ /// </summary>
+ /// <param name="key">The encryption key. Must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256 respectively.</param>
+ /// <exception cref="ArgumentException">Thrown when key length is invalid</exception>
+ public AesMessageEncryptor(byte[] key)
+ {
+ if (key.Length != 16 && key.Length != 24 && key.Length != 32)
+ {
+ throw new ArgumentException("Key must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256",
+ nameof(key));
+ }
+
+ _key = key;
+ }
+
+ /// <summary>
+ /// Encrypts the provided plain data using AES-GCM.
+ /// Format: [12-byte nonce][encrypted data][16-byte authentication tag]
+ /// </summary>
+ /// <param name="plainData">The plain data to encrypt</param>
+ /// <returns>The encrypted data with nonce and tag</returns>
+ public byte[] Encrypt(Span<byte> plainData)
+ {
+ using var aesGcm = new AesGcm(_key, TagSize);
+
+ var nonce = new byte[NonceSize];
+ RandomNumberGenerator.Fill(nonce);
+
+ var ciphertext = new byte[plainData.Length];
+ var tag = new byte[TagSize];
+
+ aesGcm.Encrypt(nonce, plainData, ciphertext, tag);
+
+ // Combine: nonce + ciphertext + tag
+ var result = new byte[NonceSize + ciphertext.Length + TagSize];
+ Buffer.BlockCopy(nonce, 0, result, 0, NonceSize);
+ Buffer.BlockCopy(ciphertext, 0, result, NonceSize, ciphertext.Length);
+ Buffer.BlockCopy(tag, 0, result, NonceSize + ciphertext.Length, TagSize);
+
+ return result;
+ }
+
+ /// <summary>
+ /// Decrypts the provided encrypted data using AES-GCM.
+ /// Expected format: [12-byte nonce][encrypted data][16-byte authentication tag]
+ /// </summary>
+ /// <param name="encryptedData">The encrypted data with nonce and tag</param>
+ /// <returns>The decrypted plain data</returns>
+ /// <exception cref="ArgumentException">Thrown when encrypted data format is invalid</exception>
+ /// <exception cref="CryptographicException">Thrown when decryption or authentication fails</exception>
+ public byte[] Decrypt(ReadOnlySpan<byte> encryptedData)
+ {
+ if (encryptedData.Length < NonceSize + TagSize)
+ {
+ throw new ArgumentException("Encrypted data is too short to contain nonce and tag", nameof(encryptedData));
+ }
+
+ using var aesGcm = new AesGcm(_key, TagSize);
+
+ // Extract nonce, ciphertext, and tag
+ Span<byte> nonce = stackalloc byte[NonceSize];
+ Span<byte> tag = stackalloc byte[TagSize];
+ var ciphertextLength = encryptedData.Length - NonceSize - TagSize;
+ Span<byte> ciphertext = stackalloc byte[ciphertextLength];
+
+ encryptedData[..NonceSize].CopyTo(nonce);
+ encryptedData.Slice(NonceSize, ciphertextLength).CopyTo(ciphertext);
+ encryptedData.Slice(NonceSize + ciphertextLength, TagSize).CopyTo(tag);
+
+ var plaintext = new byte[ciphertextLength];
+ aesGcm.Decrypt(nonce, ciphertext, tag, plaintext);
+
+ return plaintext;
+ }
+
+ /// <summary>
+ /// Creates a new AES-256 message encryptor with the specified base64-encoded key.
+ /// </summary>
+ /// <param name="base64Key">The base64-encoded encryption key</param>
+ /// <returns>A new AesMessageEncryptor instance</returns>
+ public static AesMessageEncryptor FromBase64Key(string base64Key)
+ {
+ return new AesMessageEncryptor(Convert.FromBase64String(base64Key));
+ }
+
+ /// <summary>
+ /// Generates a new random AES-256 key.
+ /// </summary>
+ /// <returns>A 32-byte random key suitable for AES-256</returns>
+ public static byte[] GenerateKey()
+ {
+ var key = new byte[32];
+ using var rng = RandomNumberGenerator.Create();
+ rng.GetBytes(key);
+ return key;
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs b/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs
index f6111f89d..f099625f0 100644
--- a/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs
+++ b/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs
@@ -1,40 +1,40 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-namespace Apache.Iggy.Encryption;
-
-/// <summary>
-/// Interface for encrypting and decrypting message payloads in Iggy messaging system.
-/// Implementations of this interface can be used with IggyPublisher and IggyConsumer
-/// to provide end-to-end encryption of message data.
-/// </summary>
-public interface IMessageEncryptor
-{
- /// <summary>
- /// Encrypts the provided plain data.
- /// </summary>
- /// <param name="plainData">The plain data to encrypt</param>
- /// <returns>The encrypted data</returns>
- byte[] Encrypt(Span<byte> plainData);
-
- /// <summary>
- /// Decrypts the provided encrypted data.
- /// </summary>
- /// <param name="encryptedData">The encrypted data to decrypt</param>
- /// <returns>The decrypted plain data</returns>
- byte[] Decrypt(ReadOnlySpan<byte> encryptedData);
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.Iggy.Encryption;
+
+/// <summary>
+/// Interface for encrypting and decrypting message payloads in Iggy messaging system.
+/// Implementations of this interface can be used with IggyPublisher and IggyConsumer
+/// to provide end-to-end encryption of message data.
+/// </summary>
+public interface IMessageEncryptor
+{
+ /// <summary>
+ /// Encrypts the provided plain data.
+ /// </summary>
+ /// <param name="plainData">The plain data to encrypt</param>
+ /// <returns>The encrypted data</returns>
+ byte[] Encrypt(Span<byte> plainData);
+
+ /// <summary>
+ /// Decrypts the provided encrypted data.
+ /// </summary>
+ /// <param name="encryptedData">The encrypted data to decrypt</param>
+ /// <returns>The decrypted plain data</returns>
+ byte[] Decrypt(ReadOnlySpan<byte> encryptedData);
+}
diff --git a/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs b/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs
index c585e1fb5..ffa5afc16 100644
--- a/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs
+++ b/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs
@@ -1,74 +1,74 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using Apache.Iggy.Consumers;
-using Apache.Iggy.IggyClient;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Publishers;
-
-namespace Apache.Iggy.Extensions;
-
-/// <summary>
-/// Extension methods for <see cref="IIggyClient" />
-/// </summary>
-public static class IggyClientExtension
-{
- /// <summary>
- /// Creates a new <see cref="IggyConsumerBuilder" /> from <see cref="IIggyClient" /> for the specified stream and
- /// topic.
- /// </summary>
- /// <param name="client">Existing iggy client</param>
- /// <param name="streamId">Stream identifier from which to consume</param>
- /// <param name="topicId">Topic identifier from which to consume</param>
- /// <param name="consumer">Consumer</param>
- /// <returns>Iggy consumer builder</returns>
- public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId,
- Identifier topicId, Consumer consumer)
- {
- return IggyConsumerBuilder.Create(client, streamId, topicId, consumer);
- }
-
- /// <summary>
- /// Creates a new <see cref="IggyConsumerBuilder{T}" /> from <see cref="IIggyClient" /> for the specified stream and
- /// topic.
- /// </summary>
- /// <param name="client">Existing iggy client</param>
- /// <param name="streamId">Stream identifier from which to consume</param>
- /// <param name="topicId">Topic identifier from which to consume</param>
- /// <param name="consumer">Consumer</param>
- /// <param name="deserializer">Optional deserializer</param>
- /// <returns>Iggy consumer builder</returns>
- public static IggyConsumerBuilder<T> CreateConsumerBuilder<T>(this IIggyClient client, Identifier streamId,
- Identifier topicId, Consumer consumer, IDeserializer<T> deserializer)
- {
- return IggyConsumerBuilder<T>.Create(client, streamId, topicId, consumer, deserializer);
- }
-
- /// <summary>
- /// Creates a new <see cref="IggyPublisherBuilder" /> from <see cref="IIggyClient" /> for the specified stream and
- /// topic.
- /// </summary>
- /// <param name="client">Existing iggy client</param>
- /// <param name="streamId">Stream identifier to publish to</param>
- /// <param name="topicId">>Topic identifier to publish to</param>
- /// <returns>Iggy publisher builder</returns>
- public static IggyPublisherBuilder CreatePublisherBuilder(this IIggyClient client, Identifier streamId,
- Identifier topicId)
- {
- return IggyPublisherBuilder.Create(client, streamId, topicId);
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Consumers;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Publishers;
+
+namespace Apache.Iggy.Extensions;
+
+/// <summary>
+/// Extension methods for <see cref="IIggyClient" />
+/// </summary>
+public static class IggyClientExtension
+{
+ /// <summary>
+ /// Creates a new <see cref="IggyConsumerBuilder" /> from <see cref="IIggyClient" /> for the specified stream and
+ /// topic.
+ /// </summary>
+ /// <param name="client">Existing iggy client</param>
+ /// <param name="streamId">Stream identifier from which to consume</param>
+ /// <param name="topicId">Topic identifier from which to consume</param>
+ /// <param name="consumer">Consumer</param>
+ /// <returns>Iggy consumer builder</returns>
+ public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId,
+ Identifier topicId, Consumer consumer)
+ {
+ return IggyConsumerBuilder.Create(client, streamId, topicId, consumer);
+ }
+
+ /// <summary>
+ /// Creates a new <see cref="IggyConsumerBuilder{T}" /> from <see cref="IIggyClient" /> for the specified stream and
+ /// topic.
+ /// </summary>
+ /// <param name="client">Existing iggy client</param>
+ /// <param name="streamId">Stream identifier from which to consume</param>
+ /// <param name="topicId">Topic identifier from which to consume</param>
+ /// <param name="consumer">Consumer</param>
+ /// <param name="deserializer">Optional deserializer</param>
+ /// <returns>Iggy consumer builder</returns>
+ public static IggyConsumerBuilder<T> CreateConsumerBuilder<T>(this IIggyClient client, Identifier streamId,
+ Identifier topicId, Consumer consumer, IDeserializer<T> deserializer)
+ {
+ return IggyConsumerBuilder<T>.Create(client, streamId, topicId, consumer, deserializer);
+ }
+
+ /// <summary>
+ /// Creates a new <see cref="IggyPublisherBuilder" /> from <see cref="IIggyClient" /> for the specified stream and
+ /// topic.
+ /// </summary>
+ /// <param name="client">Existing iggy client</param>
+ /// <param name="streamId">Stream identifier to publish to</param>
+ /// <param name="topicId">>Topic identifier to publish to</param>
+ /// <returns>Iggy publisher builder</returns>
+ public static IggyPublisherBuilder CreatePublisherBuilder(this IIggyClient client, Identifier streamId,
+ Identifier topicId)
+ {
+ return IggyPublisherBuilder.Create(client, streamId, topicId);
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs
index 1898bdbd3..418fd5a93 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs
@@ -1,85 +1,85 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Kinds;
-
-namespace Apache.Iggy.IggyClient;
-
-/// <summary>
-/// Defines methods for consuming messages from topics in an Iggy client.
-/// </summary>
-public interface IIggyConsumer
-{
- /// <summary>
- /// Polls messages from a specified topic and partition with a given polling strategy.
- /// </summary>
- /// <remarks>
- /// This method retrieves messages from a topic based on the specified consumer and polling strategy.
- /// The polling strategy determines where to start reading messages (e.g., from a specific offset, latest, earliest).
- /// If a partition ID is not specified, messages can be consumed from any partition.
- /// </remarks>
- /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
- /// <param name="topicId">The identifier of the topic to consume from (numeric ID or name).</param>
- /// <param name="partitionId">The specific partition to consume from, or null to consume from any partition.</param>
- /// <param name="consumer">The consumer identifier (group ID or member ID).</param>
- /// <param name="pollingStrategy">The strategy for determining where to start reading messages.</param>
- /// <param name="count">The maximum number of messages to retrieve.</param>
- /// <param name="autoCommit">If true, automatically commit the offset after polling.</param>
- /// <param name="token">The cancellation token to cancel the operation.</param>
- /// <returns>A task that represents the asynchronous operation and returns the polled messages.</returns>
- Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
- Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit,
- CancellationToken token = default);
-
- /// <summary>
- /// Polls messages from a specified topic and partition while renting the payload buffers from a shared pool
- /// instead of copying them into byte arrays.
- /// </summary>
- /// <remarks>
- /// The returned rental must be disposed when the caller is done reading the payload and raw header memory.
- /// Payload and raw header slices are invalidated once the rental is disposed.
- /// </remarks>
- Task<PolledMessagesRental> PollMessagesRentedAsync(Identifier streamId, Identifier topicId, uint? partitionId,
- Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit,
- CancellationToken token = default);
-
- /// <summary>
- /// Polls messages from a specified topic using a pre-constructed request.
- /// </summary>
- /// <remarks>
- /// This is a convenience method that wraps the full PollMessagesAsync method using a request object.
- /// </remarks>
- /// <param name="request">The message fetch request containing all polling parameters.</param>
- /// <param name="token">The cancellation token to cancel the operation.</param>
- /// <returns>A task that represents the asynchronous operation and returns the polled messages.</returns>
- Task<PolledMessages> PollMessagesAsync(MessageFetchRequest request, CancellationToken token = default)
- {
- return PollMessagesAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer,
- request.PollingStrategy, request.Count, request.AutoCommit, token);
- }
-
- /// <summary>
- /// Polls messages from a specified topic using a pre-constructed request while renting the payload buffers
- /// from a shared pool.
- /// </summary>
- Task<PolledMessagesRental> PollMessagesRentedAsync(MessageFetchRequest request, CancellationToken token = default)
- {
- return PollMessagesRentedAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer,
- request.PollingStrategy, request.Count, request.AutoCommit, token);
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Kinds;
+
+namespace Apache.Iggy.IggyClient;
+
+/// <summary>
+/// Defines methods for consuming messages from topics in an Iggy client.
+/// </summary>
+public interface IIggyConsumer
+{
+ /// <summary>
+ /// Polls messages from a specified topic and partition with a given polling strategy.
+ /// </summary>
+ /// <remarks>
+ /// This method retrieves messages from a topic based on the specified consumer and polling strategy.
+ /// The polling strategy determines where to start reading messages (e.g., from a specific offset, latest, earliest).
+ /// If a partition ID is not specified, messages can be consumed from any partition.
+ /// </remarks>
+ /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
+ /// <param name="topicId">The identifier of the topic to consume from (numeric ID or name).</param>
+ /// <param name="partitionId">The specific partition to consume from, or null to consume from any partition.</param>
+ /// <param name="consumer">The consumer identifier (group ID or member ID).</param>
+ /// <param name="pollingStrategy">The strategy for determining where to start reading messages.</param>
+ /// <param name="count">The maximum number of messages to retrieve.</param>
+ /// <param name="autoCommit">If true, automatically commit the offset after polling.</param>
+ /// <param name="token">The cancellation token to cancel the operation.</param>
+ /// <returns>A task that represents the asynchronous operation and returns the polled messages.</returns>
+ Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
+ Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit,
+ CancellationToken token = default);
+
+ /// <summary>
+ /// Polls messages from a specified topic and partition while renting the payload buffers from a shared pool
+ /// instead of copying them into byte arrays.
+ /// </summary>
+ /// <remarks>
+ /// The returned rental must be disposed when the caller is done reading the payload and raw header memory.
+ /// Payload and raw header slices are invalidated once the rental is disposed.
+ /// </remarks>
+ Task<PolledMessagesRental> PollMessagesRentedAsync(Identifier streamId, Identifier topicId, uint? partitionId,
+ Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit,
+ CancellationToken token = default);
+
+ /// <summary>
+ /// Polls messages from a specified topic using a pre-constructed request.
+ /// </summary>
+ /// <remarks>
+ /// This is a convenience method that wraps the full PollMessagesAsync method using a request object.
+ /// </remarks>
+ /// <param name="request">The message fetch request containing all polling parameters.</param>
+ /// <param name="token">The cancellation token to cancel the operation.</param>
+ /// <returns>A task that represents the asynchronous operation and returns the polled messages.</returns>
+ Task<PolledMessages> PollMessagesAsync(MessageFetchRequest request, CancellationToken token = default)
+ {
+ return PollMessagesAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer,
+ request.PollingStrategy, request.Count, request.AutoCommit, token);
+ }
+
+ /// <summary>
+ /// Polls messages from a specified topic using a pre-constructed request while renting the payload buffers
+ /// from a shared pool.
+ /// </summary>
+ Task<PolledMessagesRental> PollMessagesRentedAsync(MessageFetchRequest request, CancellationToken token = default)
+ {
+ return PollMessagesRentedAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer,
+ request.PollingStrategy, request.Count, request.AutoCommit, token);
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
index e7e7e77a7..9b1ce3905 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
@@ -1,862 +1,862 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Net;
-using System.Net.Http.Headers;
-using System.Net.Http.Json;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Contracts.Auth;
-using Apache.Iggy.Contracts.Http;
-using Apache.Iggy.Contracts.Http.Auth;
-using Apache.Iggy.Enums;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Mappers;
-using Apache.Iggy.Messages;
-using Apache.Iggy.StringHandlers;
-using Apache.Iggy.Utils;
-using Partitioning = Apache.Iggy.Kinds.Partitioning;
-
-namespace Apache.Iggy.IggyClient.Implementations;
-
-/// <summary>
-/// Implementation of <see cref="IIggyClient" /> that uses <see cref="HttpClient" /> to communicate with the server.
-/// </summary>
-public class HttpMessageStream : IIggyClient
-{
- private const string Context = "csharp-sdk";
-
- private readonly HttpClient _httpClient;
-
- //TODO - create mechanism for refreshing jwt token
- //TODO - replace the HttpClient with IHttpClientFactory, when implementing support for ASP.NET Core DI
- //TODO - the error handling pattern is pretty ugly, look into moving it into an extension method
- private readonly JsonSerializerOptions _jsonSerializerOptions;
-
- internal HttpMessageStream(HttpClient httpClient)
- {
- _httpClient = httpClient;
-
- _jsonSerializerOptions = new JsonSerializerOptions
- {
- PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
- Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) }
- };
- }
-
- /// <inheritdoc />
- public async Task<StreamResponse?> CreateStreamAsync(string name, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new CreateStreamRequest(name), _jsonSerializerOptions);
-
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync("/streams", data, token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<StreamResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <inheritdoc />
- public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync($"/streams/{streamId}/purge", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync($"/streams/{streamId}", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<StreamResponse?> GetStreamByIdAsync(Identifier streamId, CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync($"/streams/{streamId}", token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<StreamResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <inheritdoc />
- public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new UpdateStreamRequest(name), _jsonSerializerOptions);
-
- var data = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PutAsync($"/streams/{streamId}", data, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<StreamResponse>> GetStreamsAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/streams", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<StreamResponse>>(_jsonSerializerOptions,
- token)
- ?? Array.Empty<StreamResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<StreamResponse>();
- }
-
- /// <inheritdoc />
- public async Task<TopicResponse?> CreateTopicAsync(Identifier streamId, string name, uint partitionsCount,
- CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null,
- TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new CreateTopicRequest
- {
- Name = name,
- CompressionAlgorithm = compressionAlgorithm,
- MaxTopicSize = maxTopicSize,
- MessageExpiry = DurationHelpers.ToDuration(messageExpiry),
- PartitionsCount = partitionsCount,
- ReplicationFactor = replicationFactor
- }, _jsonSerializerOptions);
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync($"/streams/{streamId}/topics", data, token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<TopicResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <inheritdoc />
- public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name,
- CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None,
- ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new UpdateTopicRequest(name, compressionAlgorithm, maxTopicSize,
- DurationHelpers.ToDuration(messageExpiry),
- replicationFactor),
- _jsonSerializerOptions);
- var data = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}", data, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
- {
- return _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/purge", token)
- .ContinueWith(async response =>
- {
- if (!response.Result.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response.Result);
- }
- }, token);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<TopicResponse>> GetTopicsAsync(Identifier streamId,
- CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync($"/streams/{streamId}/topics", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<TopicResponse>>(_jsonSerializerOptions, token)
- ?? Array.Empty<TopicResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<TopicResponse>();
- }
-
- /// <inheritdoc />
- public async Task<TopicResponse?> GetTopicByIdAsync(Identifier streamId, Identifier topicId,
- CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}", token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<TopicResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <inheritdoc />
- public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
- IList<Message> messages,
- CancellationToken token = default)
- {
- var request = new MessageSendRequest
- {
- StreamId = streamId,
- TopicId = topicId,
- Partitioning = partitioning,
- Messages = messages
- };
- var json = JsonSerializer.Serialize(request, _jsonSerializerOptions);
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync($"/streams/{request.StreamId}/topics/{request.TopicId}/messages",
- data,
- token);
-
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync,
- CancellationToken token = default)
- {
- var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages/flush/{partitionId}/{fsync}");
-
- var response = await _httpClient.GetAsync(url, token);
-
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response, true);
- }
- }
-
- /// <inheritdoc />
- public async Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
- Consumer consumer,
- PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
- {
- var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty;
- var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages?consumer_id={consumer.ConsumerId}" +
- $"{partitionIdParam}&kind={pollingStrategy.Kind}&value={pollingStrategy.Value}&count={count}&auto_commit={autoCommit}");
-
- var response = await _httpClient.GetAsync(url, token);
- if (response.IsSuccessStatusCode)
- {
- var pollMessages = await response.Content.ReadFromJsonAsync<PolledMessages>(_jsonSerializerOptions, token)
- ?? PolledMessages.Empty;
-
- return pollMessages;
- }
-
- await HandleResponseAsync(response, true);
- return PolledMessages.Empty;
- }
-
- /// <inheritdoc />
- public async Task<PolledMessagesRental> PollMessagesRentedAsync(Identifier streamId, Identifier topicId,
- uint? partitionId,
- Consumer consumer,
- PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
- {
- var messages = await PollMessagesAsync(streamId, topicId, partitionId, consumer, pollingStrategy, count,
- autoCommit, token);
- return BinaryMapper.ToRentedMessages(messages);
- }
-
- /// <inheritdoc />
- public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset,
- uint? partitionId, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new StoreOffsetRequest(consumer, partitionId, offset),
- _jsonSerializerOptions);
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response
- = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}/consumer-offsets", data, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<OffsetResponse?> GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId,
- uint? partitionId, CancellationToken token = default)
- {
- var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty;
- var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/" +
- $"consumer-offsets?consumer_id={consumer.ConsumerId}{partitionIdParam}",
- token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<OffsetResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId,
- CancellationToken token = default)
- {
- var partitionIdParam = partitionId.HasValue ? $"?partition_id={partitionId.Value}" : string.Empty;
- var response = await _httpClient.DeleteAsync(
- $"/streams/{streamId}/topics/{topicId}/consumer-offsets/{consumer}{partitionIdParam}", token);
- await HandleResponseAsync(response);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<ConsumerGroupResponse>> GetConsumerGroupsAsync(Identifier streamId,
- Identifier topicId, CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<ConsumerGroupResponse>>(
- _jsonSerializerOptions, token)
- ?? Array.Empty<ConsumerGroupResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<ConsumerGroupResponse>();
- }
-
- /// <inheritdoc />
- public async Task<ConsumerGroupResponse?> GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId,
- Identifier groupId, CancellationToken token = default)
- {
- var response
- = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<ConsumerGroupResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task<ConsumerGroupResponse?> CreateConsumerGroupAsync(Identifier streamId, Identifier topicId,
- string name, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new CreateConsumerGroupRequest(name), _jsonSerializerOptions);
-
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response
- = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", data, token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<ConsumerGroupResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- var response
- = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token);
- await HandleResponseAsync(response);
- }
-
- /// <summary>
- /// This method is only supported in TCP protocol
- /// </summary>
- /// <param name="token"></param>
- /// <returns></returns>
- /// <exception cref="FeatureUnavailableException"></exception>
- public Task<ClientResponse?> GetMeAsync(CancellationToken token = default)
- {
- throw new FeatureUnavailableException();
- }
-
- /// <inheritdoc />
- public async Task<StatsResponse?> GetStatsAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/stats", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<StatsResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task<ClusterMetadata?> GetClusterMetadataAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/cluster/metadata", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<ClusterMetadata>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <inheritdoc />
- public async Task PingAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/ping", token);
-
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<byte[]> GetSnapshotAsync(SnapshotCompression compression,
- IList<SystemSnapshotType> snapshotTypes, CancellationToken token = default)
- {
- // Rust serde uses default derive (PascalCase) for these enums, not snake_case.
- // We use .ToString() to produce PascalCase names matching Rust's serde expectations.
- var request = new
- {
- compression = compression.ToString(),
- snapshot_types = snapshotTypes.Select(t => t.ToString()).ToList()
- };
- var json = JsonSerializer.Serialize(request, _jsonSerializerOptions);
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync("/snapshot", data, token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadAsByteArrayAsync(token);
- }
-
- await HandleResponseAsync(response);
- return [];
- }
-
- /// <inheritdoc />
- public Task ConnectAsync(CancellationToken token = default)
- {
- return Task.CompletedTask;
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<ClientResponse>> GetClientsAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/clients", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<ClientResponse>>(_jsonSerializerOptions,
- token)
- ?? Array.Empty<ClientResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<ClientResponse>();
- }
-
- /// <inheritdoc />
- public async Task<ClientResponse?> GetClientByIdAsync(uint clientId, CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync($"/clients/{clientId}", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<ClientResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <summary>
- /// This method is only supported in TCP protocol
- /// </summary>
- /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
- /// <param name="topicId">The identifier of the topic (numeric ID or name).</param>
- /// <param name="groupId">The identifier of the consumer group to join (numeric ID or name).</param>
- /// <param name="token">The cancellation token to cancel the operation.</param>
- /// <returns>A task representing the asynchronous operation.</returns>
- /// <exception cref="FeatureUnavailableException"></exception>
- [Obsolete("This method is only supported in TCP protocol", true)]
- public Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- throw new FeatureUnavailableException();
- }
-
- /// <summary>
- /// This method is only supported in TCP protocol
- /// </summary>
- /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
- /// <param name="topicId">The identifier of the topic (numeric ID or name).</param>
- /// <param name="groupId">The identifier of the consumer group to leave (numeric ID or name).</param>
- /// <param name="token">The cancellation token to cancel the operation.</param>
- /// <returns>A task representing the asynchronous operation.</returns>
- /// <exception cref="FeatureUnavailableException"></exception>
- [Obsolete("This method is only supported in TCP protocol", true)]
- public Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- throw new FeatureUnavailableException();
- }
-
- /// <inheritdoc />
- public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
- CancellationToken token = default)
- {
- var response
- = await _httpClient.DeleteAsync(
- $"/streams/{streamId}/topics/{topicId}/partitions?partitions_count={partitionsCount}", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <summary>
- /// This method is only supported in TCP protocol
- /// </summary>
- /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
- /// <param name="topicId">The identifier of the topic containing the partition (numeric ID or name).</param>
- /// <param name="partitionId">The unique partition ID.</param>
- /// <param name="segmentsCount">The number of segments to delete.</param>
- /// <param name="token">The cancellation token to cancel the operation.</param>
- /// <returns>A task representing the asynchronous operation.</returns>
- /// <exception cref="FeatureUnavailableException"></exception>
- public Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId,
- uint segmentsCount, CancellationToken token = default)
- {
- throw new FeatureUnavailableException();
- }
-
- /// <inheritdoc />
- public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new CreatePartitionsRequest(partitionsCount), _jsonSerializerOptions);
-
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/partitions", data, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<UserResponse?> GetUserAsync(Identifier userId, CancellationToken token = default)
- {
- //TODO - this doesn't work prob needs a custom json serializer
- var response = await _httpClient.GetAsync($"/users/{userId}", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<UserResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<UserResponse>> GetUsersAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/users", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<UserResponse>>(_jsonSerializerOptions, token)
- ?? Array.Empty<UserResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<UserResponse>();
- }
-
- /// <inheritdoc />
- public async Task<UserResponse?> CreateUserAsync(string userName, string password, UserStatus status,
- Permissions? permissions = null, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new CreateUserRequest(userName, password, status, permissions),
- _jsonSerializerOptions);
-
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PostAsync("/users", content, token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<UserResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync($"/users/{userId}", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new UpdateUserRequest(userName, status), _jsonSerializerOptions);
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PutAsync($"/users/{userId}", content, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new UpdateUserPermissionsRequest(permissions), _jsonSerializerOptions);
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PutAsync($"/users/{userId}/permissions", content, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new ChangePasswordRequest(currentPassword, newPassword),
- _jsonSerializerOptions);
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PutAsync($"/users/{userId}/password", content, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<AuthResponse?> LoginUserAsync(string userName, string password, CancellationToken token = default)
- {
- // TODO: Add binary protocol version
- var json = JsonSerializer.Serialize(new LoginUserRequest(userName, password, SdkVersion.Value, Context),
- _jsonSerializerOptions);
-
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync("users/login", data, token);
- if (response.IsSuccessStatusCode)
- {
- var authResponse = await response.Content.ReadFromJsonAsync<AuthResponse>(_jsonSerializerOptions, token);
- var jwtToken = authResponse!.AccessToken?.Token;
- if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token))
- {
- _httpClient.DefaultRequestHeaders.Authorization =
- new AuthenticationHeaderValue("Bearer", jwtToken);
- }
- else
- {
- throw new Exception("The JWT token is missing.");
- }
-
- return authResponse;
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task LogoutUserAsync(CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync("users/logout", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
-
- _httpClient.DefaultRequestHeaders.Authorization = null;
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<PersonalAccessTokenResponse>> GetPersonalAccessTokensAsync(
- CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/personal-access-tokens", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<PersonalAccessTokenResponse>>(
- _jsonSerializerOptions, token)
- ?? Array.Empty<PersonalAccessTokenResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<PersonalAccessTokenResponse>();
- }
-
- /// <inheritdoc />
- public async Task<RawPersonalAccessToken?> CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(
- new CreatePersonalAccessTokenRequest(name, DurationHelpers.ToDuration(expiry)), _jsonSerializerOptions);
-
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PostAsync("/personal-access-tokens", content, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
-
- return await response.Content.ReadFromJsonAsync<RawPersonalAccessToken>(_jsonSerializerOptions, token);
- }
-
- /// <inheritdoc />
- public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync($"/personal-access-tokens/{name}", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<AuthResponse?> LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default)
- {
- var json = JsonSerializer.Serialize(new LoginWithPersonalAccessTokenRequest(token), _jsonSerializerOptions);
-
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PostAsync("/personal-access-tokens/login", content, ct);
- if (response.IsSuccessStatusCode)
- {
- var authResponse = await response.Content.ReadFromJsonAsync<AuthResponse>(_jsonSerializerOptions, ct);
- var jwtToken = authResponse!.AccessToken?.Token;
- if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token))
- {
- _httpClient.DefaultRequestHeaders.Authorization =
- new AuthenticationHeaderValue("Bearer", jwtToken);
- }
- else
- {
- throw new Exception("The JWT token is missing.");
- }
-
- return authResponse;
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <summary>
- /// Dispose the client.
- /// </summary>
- public void Dispose()
- {
- }
-
- /// <inheritdoc />
- public void SubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
- {
- }
-
- /// <inheritdoc />
- public void UnsubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
- {
- }
-
- /// <inheritdoc />
- public string GetCurrentAddress()
- {
- return _httpClient.BaseAddress?.ToString() ?? string.Empty;
- }
-
- private static async Task HandleResponseAsync(HttpResponseMessage response, bool shouldThrowOnGetNotFound = false)
- {
- if ((int)response.StatusCode > 300
- && (int)response.StatusCode < 500
- && !(response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound &&
- !shouldThrowOnGetNotFound))
- {
- var err = await response.Content.ReadAsStringAsync();
- var errorModel = JsonSerializer.Deserialize<ErrorResponse>(err);
- throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err);
- }
-
- if (response.StatusCode == HttpStatusCode.InternalServerError)
- {
- throw new Exception("Internal server error");
- }
- }
-
- private static string CreateUrl(ref MessageRequestInterpolationHandler message)
- {
- return message.ToString();
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Net;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Contracts.Auth;
+using Apache.Iggy.Contracts.Http;
+using Apache.Iggy.Contracts.Http.Auth;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Mappers;
+using Apache.Iggy.Messages;
+using Apache.Iggy.StringHandlers;
+using Apache.Iggy.Utils;
+using Partitioning = Apache.Iggy.Kinds.Partitioning;
+
+namespace Apache.Iggy.IggyClient.Implementations;
+
+/// <summary>
+/// Implementation of <see cref="IIggyClient" /> that uses <see cref="HttpClient" /> to communicate with the server.
+/// </summary>
+public class HttpMessageStream : IIggyClient
+{
+ private const string Context = "csharp-sdk";
+
+ private readonly HttpClient _httpClient;
+
+ //TODO - create mechanism for refreshing jwt token
+ //TODO - replace the HttpClient with IHttpClientFactory, when implementing support for ASP.NET Core DI
+ //TODO - the error handling pattern is pretty ugly, look into moving it into an extension method
+ private readonly JsonSerializerOptions _jsonSerializerOptions;
+
+ internal HttpMessageStream(HttpClient httpClient)
+ {
+ _httpClient = httpClient;
+
+ _jsonSerializerOptions = new JsonSerializerOptions
+ {
+ PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
+ Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) }
+ };
+ }
+
+ /// <inheritdoc />
+ public async Task<StreamResponse?> CreateStreamAsync(string name, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new CreateStreamRequest(name), _jsonSerializerOptions);
+
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync("/streams", data, token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<StreamResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync($"/streams/{streamId}/purge", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync($"/streams/{streamId}", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<StreamResponse?> GetStreamByIdAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync($"/streams/{streamId}", token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<StreamResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new UpdateStreamRequest(name), _jsonSerializerOptions);
+
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PutAsync($"/streams/{streamId}", data, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<StreamResponse>> GetStreamsAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/streams", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<StreamResponse>>(_jsonSerializerOptions,
+ token)
+ ?? Array.Empty<StreamResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<StreamResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<TopicResponse?> CreateTopicAsync(Identifier streamId, string name, uint partitionsCount,
+ CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null,
+ TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new CreateTopicRequest
+ {
+ Name = name,
+ CompressionAlgorithm = compressionAlgorithm,
+ MaxTopicSize = maxTopicSize,
+ MessageExpiry = DurationHelpers.ToDuration(messageExpiry),
+ PartitionsCount = partitionsCount,
+ ReplicationFactor = replicationFactor
+ }, _jsonSerializerOptions);
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync($"/streams/{streamId}/topics", data, token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<TopicResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name,
+ CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None,
+ ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new UpdateTopicRequest(name, compressionAlgorithm, maxTopicSize,
+ DurationHelpers.ToDuration(messageExpiry),
+ replicationFactor),
+ _jsonSerializerOptions);
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}", data, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
+ {
+ return _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/purge", token)
+ .ContinueWith(async response =>
+ {
+ if (!response.Result.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response.Result);
+ }
+ }, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<TopicResponse>> GetTopicsAsync(Identifier streamId,
+ CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync($"/streams/{streamId}/topics", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<TopicResponse>>(_jsonSerializerOptions, token)
+ ?? Array.Empty<TopicResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<TopicResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<TopicResponse?> GetTopicByIdAsync(Identifier streamId, Identifier topicId,
+ CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}", token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<TopicResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
+ IList<Message> messages,
+ CancellationToken token = default)
+ {
+ var request = new MessageSendRequest
+ {
+ StreamId = streamId,
+ TopicId = topicId,
+ Partitioning = partitioning,
+ Messages = messages
+ };
+ var json = JsonSerializer.Serialize(request, _jsonSerializerOptions);
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync($"/streams/{request.StreamId}/topics/{request.TopicId}/messages",
+ data,
+ token);
+
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync,
+ CancellationToken token = default)
+ {
+ var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages/flush/{partitionId}/{fsync}");
+
+ var response = await _httpClient.GetAsync(url, token);
+
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response, true);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
+ Consumer consumer,
+ PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
+ {
+ var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty;
+ var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages?consumer_id={consumer.ConsumerId}" +
+ $"{partitionIdParam}&kind={pollingStrategy.Kind}&value={pollingStrategy.Value}&count={count}&auto_commit={autoCommit}");
+
+ var response = await _httpClient.GetAsync(url, token);
+ if (response.IsSuccessStatusCode)
+ {
+ var pollMessages = await response.Content.ReadFromJsonAsync<PolledMessages>(_jsonSerializerOptions, token)
+ ?? PolledMessages.Empty;
+
+ return pollMessages;
+ }
+
+ await HandleResponseAsync(response, true);
+ return PolledMessages.Empty;
+ }
+
+ /// <inheritdoc />
+ public async Task<PolledMessagesRental> PollMessagesRentedAsync(Identifier streamId, Identifier topicId,
+ uint? partitionId,
+ Consumer consumer,
+ PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
+ {
+ var messages = await PollMessagesAsync(streamId, topicId, partitionId, consumer, pollingStrategy, count,
+ autoCommit, token);
+ return BinaryMapper.ToRentedMessages(messages);
+ }
+
+ /// <inheritdoc />
+ public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset,
+ uint? partitionId, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new StoreOffsetRequest(consumer, partitionId, offset),
+ _jsonSerializerOptions);
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response
+ = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}/consumer-offsets", data, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<OffsetResponse?> GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId,
+ uint? partitionId, CancellationToken token = default)
+ {
+ var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty;
+ var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/" +
+ $"consumer-offsets?consumer_id={consumer.ConsumerId}{partitionIdParam}",
+ token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<OffsetResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId,
+ CancellationToken token = default)
+ {
+ var partitionIdParam = partitionId.HasValue ? $"?partition_id={partitionId.Value}" : string.Empty;
+ var response = await _httpClient.DeleteAsync(
+ $"/streams/{streamId}/topics/{topicId}/consumer-offsets/{consumer}{partitionIdParam}", token);
+ await HandleResponseAsync(response);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<ConsumerGroupResponse>> GetConsumerGroupsAsync(Identifier streamId,
+ Identifier topicId, CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<ConsumerGroupResponse>>(
+ _jsonSerializerOptions, token)
+ ?? Array.Empty<ConsumerGroupResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<ConsumerGroupResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<ConsumerGroupResponse?> GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId,
+ Identifier groupId, CancellationToken token = default)
+ {
+ var response
+ = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<ConsumerGroupResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task<ConsumerGroupResponse?> CreateConsumerGroupAsync(Identifier streamId, Identifier topicId,
+ string name, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new CreateConsumerGroupRequest(name), _jsonSerializerOptions);
+
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response
+ = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", data, token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<ConsumerGroupResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ var response
+ = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token);
+ await HandleResponseAsync(response);
+ }
+
+ /// <summary>
+ /// This method is only supported in TCP protocol
+ /// </summary>
+ /// <param name="token"></param>
+ /// <returns></returns>
+ /// <exception cref="FeatureUnavailableException"></exception>
+ public Task<ClientResponse?> GetMeAsync(CancellationToken token = default)
+ {
+ throw new FeatureUnavailableException();
+ }
+
+ /// <inheritdoc />
+ public async Task<StatsResponse?> GetStatsAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/stats", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<StatsResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task<ClusterMetadata?> GetClusterMetadataAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/cluster/metadata", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<ClusterMetadata>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task PingAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/ping", token);
+
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<byte[]> GetSnapshotAsync(SnapshotCompression compression,
+ IList<SystemSnapshotType> snapshotTypes, CancellationToken token = default)
+ {
+ // Rust serde uses default derive (PascalCase) for these enums, not snake_case.
+ // We use .ToString() to produce PascalCase names matching Rust's serde expectations.
+ var request = new
+ {
+ compression = compression.ToString(),
+ snapshot_types = snapshotTypes.Select(t => t.ToString()).ToList()
+ };
+ var json = JsonSerializer.Serialize(request, _jsonSerializerOptions);
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync("/snapshot", data, token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadAsByteArrayAsync(token);
+ }
+
+ await HandleResponseAsync(response);
+ return [];
+ }
+
+ /// <inheritdoc />
+ public Task ConnectAsync(CancellationToken token = default)
+ {
+ return Task.CompletedTask;
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<ClientResponse>> GetClientsAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/clients", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<ClientResponse>>(_jsonSerializerOptions,
+ token)
+ ?? Array.Empty<ClientResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<ClientResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<ClientResponse?> GetClientByIdAsync(uint clientId, CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync($"/clients/{clientId}", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<ClientResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <summary>
+ /// This method is only supported in TCP protocol
+ /// </summary>
+ /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
+ /// <param name="topicId">The identifier of the topic (numeric ID or name).</param>
+ /// <param name="groupId">The identifier of the consumer group to join (numeric ID or name).</param>
+ /// <param name="token">The cancellation token to cancel the operation.</param>
+ /// <returns>A task representing the asynchronous operation.</returns>
+ /// <exception cref="FeatureUnavailableException"></exception>
+ [Obsolete("This method is only supported in TCP protocol", true)]
+ public Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ throw new FeatureUnavailableException();
+ }
+
+ /// <summary>
+ /// This method is only supported in TCP protocol
+ /// </summary>
+ /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
+ /// <param name="topicId">The identifier of the topic (numeric ID or name).</param>
+ /// <param name="groupId">The identifier of the consumer group to leave (numeric ID or name).</param>
+ /// <param name="token">The cancellation token to cancel the operation.</param>
+ /// <returns>A task representing the asynchronous operation.</returns>
+ /// <exception cref="FeatureUnavailableException"></exception>
+ [Obsolete("This method is only supported in TCP protocol", true)]
+ public Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ throw new FeatureUnavailableException();
+ }
+
+ /// <inheritdoc />
+ public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
+ CancellationToken token = default)
+ {
+ var response
+ = await _httpClient.DeleteAsync(
+ $"/streams/{streamId}/topics/{topicId}/partitions?partitions_count={partitionsCount}", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <summary>
+ /// This method is only supported in TCP protocol
+ /// </summary>
+ /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
+ /// <param name="topicId">The identifier of the topic containing the partition (numeric ID or name).</param>
+ /// <param name="partitionId">The unique partition ID.</param>
+ /// <param name="segmentsCount">The number of segments to delete.</param>
+ /// <param name="token">The cancellation token to cancel the operation.</param>
+ /// <returns>A task representing the asynchronous operation.</returns>
+ /// <exception cref="FeatureUnavailableException"></exception>
+ public Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId,
+ uint segmentsCount, CancellationToken token = default)
+ {
+ throw new FeatureUnavailableException();
+ }
+
+ /// <inheritdoc />
+ public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new CreatePartitionsRequest(partitionsCount), _jsonSerializerOptions);
+
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/partitions", data, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<UserResponse?> GetUserAsync(Identifier userId, CancellationToken token = default)
+ {
+ //TODO - this doesn't work prob needs a custom json serializer
+ var response = await _httpClient.GetAsync($"/users/{userId}", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<UserResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<UserResponse>> GetUsersAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/users", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<UserResponse>>(_jsonSerializerOptions, token)
+ ?? Array.Empty<UserResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<UserResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<UserResponse?> CreateUserAsync(string userName, string password, UserStatus status,
+ Permissions? permissions = null, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new CreateUserRequest(userName, password, status, permissions),
+ _jsonSerializerOptions);
+
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PostAsync("/users", content, token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<UserResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync($"/users/{userId}", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new UpdateUserRequest(userName, status), _jsonSerializerOptions);
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PutAsync($"/users/{userId}", content, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new UpdateUserPermissionsRequest(permissions), _jsonSerializerOptions);
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PutAsync($"/users/{userId}/permissions", content, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new ChangePasswordRequest(currentPassword, newPassword),
+ _jsonSerializerOptions);
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PutAsync($"/users/{userId}/password", content, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<AuthResponse?> LoginUserAsync(string userName, string password, CancellationToken token = default)
+ {
+ // TODO: Add binary protocol version
+ var json = JsonSerializer.Serialize(new LoginUserRequest(userName, password, SdkVersion.Value, Context),
+ _jsonSerializerOptions);
+
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync("users/login", data, token);
+ if (response.IsSuccessStatusCode)
+ {
+ var authResponse = await response.Content.ReadFromJsonAsync<AuthResponse>(_jsonSerializerOptions, token);
+ var jwtToken = authResponse!.AccessToken?.Token;
+ if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token))
+ {
+ _httpClient.DefaultRequestHeaders.Authorization =
+ new AuthenticationHeaderValue("Bearer", jwtToken);
+ }
+ else
+ {
+ throw new Exception("The JWT token is missing.");
+ }
+
+ return authResponse;
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task LogoutUserAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync("users/logout", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+
+ _httpClient.DefaultRequestHeaders.Authorization = null;
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<PersonalAccessTokenResponse>> GetPersonalAccessTokensAsync(
+ CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/personal-access-tokens", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<PersonalAccessTokenResponse>>(
+ _jsonSerializerOptions, token)
+ ?? Array.Empty<PersonalAccessTokenResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<PersonalAccessTokenResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<RawPersonalAccessToken?> CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(
+ new CreatePersonalAccessTokenRequest(name, DurationHelpers.ToDuration(expiry)), _jsonSerializerOptions);
+
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PostAsync("/personal-access-tokens", content, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+
+ return await response.Content.ReadFromJsonAsync<RawPersonalAccessToken>(_jsonSerializerOptions, token);
+ }
+
+ /// <inheritdoc />
+ public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync($"/personal-access-tokens/{name}", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<AuthResponse?> LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default)
+ {
+ var json = JsonSerializer.Serialize(new LoginWithPersonalAccessTokenRequest(token), _jsonSerializerOptions);
+
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PostAsync("/personal-access-tokens/login", content, ct);
+ if (response.IsSuccessStatusCode)
+ {
+ var authResponse = await response.Content.ReadFromJsonAsync<AuthResponse>(_jsonSerializerOptions, ct);
+ var jwtToken = authResponse!.AccessToken?.Token;
+ if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token))
+ {
+ _httpClient.DefaultRequestHeaders.Authorization =
+ new AuthenticationHeaderValue("Bearer", jwtToken);
+ }
+ else
+ {
+ throw new Exception("The JWT token is missing.");
+ }
+
+ return authResponse;
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <summary>
+ /// Dispose the client.
+ /// </summary>
+ public void Dispose()
+ {
+ }
+
+ /// <inheritdoc />
+ public void SubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
+ {
+ }
+
+ /// <inheritdoc />
+ public void UnsubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
+ {
+ }
+
+ /// <inheritdoc />
+ public string GetCurrentAddress()
+ {
+ return _httpClient.BaseAddress?.ToString() ?? string.Empty;
+ }
+
+ private static async Task HandleResponseAsync(HttpResponseMessage response, bool shouldThrowOnGetNotFound = false)
+ {
+ if ((int)response.StatusCode > 300
+ && (int)response.StatusCode < 500
+ && !(response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound &&
+ !shouldThrowOnGetNotFound))
+ {
+ var err = await response.Content.ReadAsStringAsync();
+ var errorModel = JsonSerializer.Deserialize<ErrorResponse>(err);
+ throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err);
+ }
+
+ if (response.StatusCode == HttpStatusCode.InternalServerError)
+ {
+ throw new Exception("Internal server error");
+ }
+ }
+
+ private static string CreateUrl(ref MessageRequestInterpolationHandler message)
+ {
+ return message.ToString();
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
index 72d0d9b51..8a9d2b58b 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
@@ -1,1317 +1,1317 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Buffers;
-using System.Buffers.Binary;
-using System.Net.Security;
-using System.Net.Sockets;
-using System.Security.Cryptography.X509Certificates;
-using System.Text;
-using Apache.Iggy.Configuration;
-using Apache.Iggy.ConnectionStream;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Contracts.Auth;
-using Apache.Iggy.Contracts.Tcp;
-using Apache.Iggy.Enums;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Mappers;
-using Apache.Iggy.Messages;
-using Apache.Iggy.Utils;
-using Microsoft.Extensions.Logging;
-using Partitioning = Apache.Iggy.Kinds.Partitioning;
-
-namespace Apache.Iggy.IggyClient.Implementations;
-
-/// <summary>
-/// A TCP client for interacting with the Iggy server.
-/// </summary>
-public sealed class TcpMessageStream : IIggyClient
-{
- private readonly IggyClientConfigurator _configuration;
- private readonly EventAggregator<ConnectionStateChangedEventArgs> _connectionEvents;
- private readonly SemaphoreSlim _connectionSemaphore;
- private readonly ILogger<TcpMessageStream> _logger;
- private readonly byte[] _responseHeaderBuffer = new byte[BufferSizes.EXPECTED_RESPONSE_SIZE];
- private readonly SemaphoreSlim _sendingSemaphore;
- private string _currentAddress = string.Empty;
- private X509Certificate2Collection _customCaStore = [];
- private bool _isConnecting;
- private DateTimeOffset _lastConnectionTime;
- private ConnectionState _state = ConnectionState.Disconnected;
- private TcpConnectionStream _stream = null!;
-
- internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory loggerFactory)
- {
- _configuration = configuration;
- _logger = loggerFactory.CreateLogger<TcpMessageStream>();
- _sendingSemaphore = new SemaphoreSlim(1, 1);
- _connectionSemaphore = new SemaphoreSlim(1, 1);
- _lastConnectionTime = DateTimeOffset.MinValue;
- _connectionEvents = new EventAggregator<ConnectionStateChangedEventArgs>(loggerFactory);
- }
-
- /// <summary>
- /// Fired whenever the connection state changes.
- /// </summary>
- //public event EventHandler<ConnectionStateChangedEventArgs>? OnConnectionStateChanged;
- public void Dispose()
- {
- _stream?.Close();
- _stream?.Dispose();
- _sendingSemaphore.Dispose();
- _connectionSemaphore.Dispose();
- _connectionEvents.Clear();
- }
-
- /// <inheritdoc />
- public void SubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
- {
- _connectionEvents.Subscribe(callback);
- }
-
- /// <inheritdoc />
- public void UnsubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
- {
- _connectionEvents.Unsubscribe(callback);
- }
-
- /// <inheritdoc />
- public string GetCurrentAddress()
- {
- return _currentAddress;
- }
-
- /// <inheritdoc />
- public async Task<StreamResponse?> CreateStreamAsync(string name, CancellationToken token = default)
- {
- var message = TcpContracts.CreateStream(name);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_STREAM_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- throw new InvalidResponseException("Received empty response while trying to create stream.");
- }
-
- return BinaryMapper.MapStream(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<StreamResponse?> GetStreamByIdAsync(Identifier streamId, CancellationToken token = default)
- {
- var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAM_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapStream(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<StreamResponse>> GetStreamsAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAMS_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapStreams(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default)
- {
- var message = TcpContracts.UpdateStream(streamId, name);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_STREAM_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default)
- {
- var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_STREAM_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default)
- {
- var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_STREAM_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<TopicResponse>> GetTopicsAsync(Identifier streamId,
- CancellationToken token = default)
- {
- var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPICS_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapTopics(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<TopicResponse?> GetTopicByIdAsync(Identifier streamId, Identifier topicId,
- CancellationToken token = default)
- {
- var message = TcpContracts.GetTopicById(streamId, topicId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPIC_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapTopic(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<TopicResponse?> CreateTopicAsync(Identifier streamId, string name, uint partitionsCount,
- CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null,
- TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default)
- {
- var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry);
- var message = TcpContracts.CreateTopic(streamId, name, partitionsCount, compressionAlgorithm,
- replicationFactor, messageExpiryValue, maxTopicSize);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_TOPIC_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapTopic(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name,
- CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None,
- ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null,
- CancellationToken token = default)
- {
- var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry);
- var message = TcpContracts.UpdateTopic(streamId, topicId, name, compressionAlgorithm, maxTopicSize,
- messageExpiryValue, replicationFactor);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_TOPIC_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
- {
- var message = TcpContracts.DeleteTopic(streamId, topicId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_TOPIC_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
- {
- var message = TcpContracts.PurgeTopic(streamId, topicId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_TOPIC_CODE);
-
- await SendAckAsync(payload, token);
- }
-
-
- /// <inheritdoc />
- public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
- IList<Message> messages, CancellationToken token = default)
- {
- var metadataLength = 2 + streamId.Length + 2 + topicId.Length
- + 2 + partitioning.Length + 4 + 4;
- var messageBufferSize = TcpMessageStreamHelpers.CalculateMessageBytesCount(messages)
- + metadataLength;
- var payloadBufferSize = messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH;
-
- IMemoryOwner<byte> messageBuffer = MemoryPool<byte>.Shared.Rent(messageBufferSize);
- IMemoryOwner<byte> payloadBuffer = MemoryPool<byte>.Shared.Rent(payloadBufferSize);
- try
- {
- TcpContracts.CreateMessage(messageBuffer.Memory.Span[..messageBufferSize], streamId,
- topicId, partitioning, messages);
-
- TcpMessageStreamHelpers.CreatePayload(payloadBuffer.Memory.Span[..payloadBufferSize],
- messageBuffer.Memory.Span[..messageBufferSize], CommandCodes.SEND_MESSAGES_CODE);
-
- await SendAckAsync(payloadBuffer.Memory[..payloadBufferSize], token);
- }
- finally
- {
- messageBuffer.Dispose();
- payloadBuffer.Dispose();
- }
- }
-
- /// <inheritdoc />
- public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync,
- CancellationToken token = default)
- {
- var message = TcpContracts.FlushUnsavedBuffer(streamId, topicId, partitionId, fsync);
-
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.FLUSH_UNSAVED_BUFFER_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
- Consumer consumer,
- PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
- {
- using var rental = await PollMessagesRentedAsync(streamId, topicId, partitionId, consumer, pollingStrategy,
- count, autoCommit, token);
- return BinaryMapper.MaterializeMessages(rental);
- }
-
- /// <inheritdoc />
- public async Task<PolledMessagesRental> PollMessagesRentedAsync(Identifier streamId, Identifier topicId,
- uint? partitionId,
- Consumer consumer,
- PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
- {
- var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer);
- var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize);
- var payload = ArrayPool<byte>.Shared.Rent(payloadBufferSize);
-
- try
- {
- TcpContracts.GetMessages(payload.AsSpan().Slice(8, messageBufferSize), consumer, streamId,
- topicId, pollingStrategy, count, autoCommit, partitionId);
- BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[..4], messageBufferSize + 4);
- BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[4..8], CommandCodes.POLL_MESSAGES_CODE);
-
- IMemoryOwner<byte> responseBuffer
- = await SendWithResponseAsync(payload.AsMemory(0, payloadBufferSize), token);
- return BinaryMapper.MapRentedMessages(responseBuffer);
- }
- finally
- {
- ArrayPool<byte>.Shared.Return(payload);
- }
- }
-
- /// <inheritdoc />
- public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset,
- uint? partitionId, CancellationToken token = default)
- {
- var message = TcpContracts.UpdateOffset(streamId, topicId, consumer, offset, partitionId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.STORE_CONSUMER_OFFSET_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<OffsetResponse?> GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId,
- uint? partitionId, CancellationToken token = default)
- {
- var message = TcpContracts.GetOffset(streamId, topicId, consumer, partitionId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_OFFSET_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapOffsets(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId,
- CancellationToken token = default)
- {
- var message = TcpContracts.DeleteOffset(streamId, topicId, consumer, partitionId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_OFFSET_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<ConsumerGroupResponse>> GetConsumerGroupsAsync(Identifier streamId,
- Identifier topicId,
- CancellationToken token = default)
- {
- var message = TcpContracts.GetGroups(streamId, topicId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUPS_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapConsumerGroups(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<ConsumerGroupResponse?> GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId,
- Identifier groupId, CancellationToken token = default)
- {
- var message = TcpContracts.GetGroup(streamId, topicId, groupId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUP_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<ConsumerGroupResponse?> CreateConsumerGroupAsync(Identifier streamId, Identifier topicId,
- string name, CancellationToken token = default)
- {
- var message = TcpContracts.CreateGroup(streamId, topicId, name);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_CONSUMER_GROUP_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- var message = TcpContracts.DeleteGroup(streamId, topicId, groupId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_GROUP_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- var message = TcpContracts.JoinGroup(streamId, topicId, groupId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.JOIN_CONSUMER_GROUP_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- var message = TcpContracts.LeaveGroup(streamId, topicId, groupId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LEAVE_CONSUMER_GROUP_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
- CancellationToken token = default)
- {
- var message = TcpContracts.DeletePartitions(streamId, topicId, partitionsCount);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PARTITIONS_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
- CancellationToken token = default)
- {
- var message = TcpContracts.CreatePartitions(streamId, topicId, partitionsCount);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PARTITIONS_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId,
- uint segmentsCount, CancellationToken token = default)
- {
- var message = TcpContracts.DeleteSegments(streamId, topicId, partitionId, segmentsCount);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_SEGMENTS_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<ClientResponse?> GetMeAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_ME_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapClient(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<StatsResponse?> GetStatsAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STATS_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapStats(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<ClusterMetadata?> GetClusterMetadataAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLUSTER_METADATA_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapClusterMetadata(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task PingAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PING_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<byte[]> GetSnapshotAsync(SnapshotCompression compression,
- IList<SystemSnapshotType> snapshotTypes, CancellationToken token = default)
- {
- var message = TcpContracts.GetSnapshot(compression, snapshotTypes);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_SNAPSHOT_CODE);
-
- using IMemoryOwner<byte> result = await SendWithResponseAsync(payload, token);
-
- return result.Memory.Span.ToArray();
- }
-
- /// <inheritdoc />
- public async Task ConnectAsync(CancellationToken token = default)
- {
- if (_state is ConnectionState.Connected
- or ConnectionState.Authenticating
- or ConnectionState.Authenticated)
- {
- _logger.LogWarning("Connection is already connected");
- return;
- }
-
- if (_lastConnectionTime != DateTimeOffset.MinValue)
- {
- await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token);
- }
-
- SetConnectionStateAsync(ConnectionState.Connecting);
- _isConnecting = true;
- try
- {
- await TryEstablishConnectionAsync(token);
- }
- finally
- {
- _isConnecting = false;
- }
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<ClientResponse>> GetClientsAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENTS_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapClients(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<ClientResponse?> GetClientByIdAsync(uint clientId, CancellationToken token = default)
- {
- var message = TcpContracts.GetClient(clientId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENT_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapClient(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<UserResponse?> GetUserAsync(Identifier userId, CancellationToken token = default)
- {
- var message = TcpContracts.GetUser(userId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USER_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapUser(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<UserResponse>> GetUsersAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USERS_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapUsers(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<UserResponse?> CreateUserAsync(string userName, string password, UserStatus status,
- Permissions? permissions = null, CancellationToken token = default)
- {
- var message = TcpContracts.CreateUser(userName, password, status, permissions);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_USER_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapUser(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default)
- {
- var message = TcpContracts.DeleteUser(userId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_USER_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null,
- CancellationToken token = default)
- {
- var message = TcpContracts.UpdateUser(userId, userName, status);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_USER_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null,
- CancellationToken token = default)
- {
- var message = TcpContracts.UpdatePermissions(userId, permissions);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_PERMISSIONS_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword,
- CancellationToken token = default)
- {
- var message = TcpContracts.ChangePassword(userId, currentPassword, newPassword);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CHANGE_PASSWORD_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<AuthResponse?> LoginUserAsync(string userName, string password, CancellationToken token = default)
- {
- if (_state == ConnectionState.Disconnected)
- {
- throw new NotConnectedException();
- }
-
- // TODO: Add binary protocol version
- var message = TcpContracts.LoginUser(userName, password, SdkVersion.Value, "csharp-sdk");
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_USER_CODE);
-
- SetConnectionStateAsync(ConnectionState.Authenticating);
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..responseBuffer.Memory.Length]);
- SetConnectionStateAsync(ConnectionState.Authenticated);
-
- if (await RedirectAsync(token))
- {
- await ConnectAsync(token);
- return await LoginUserAsync(userName, password, token);
- }
-
- var authResponse = new AuthResponse(userId, null);
- return authResponse;
- }
-
- /// <inheritdoc />
- public async Task LogoutUserAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGOUT_USER_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<PersonalAccessTokenResponse>> GetPersonalAccessTokensAsync(
- CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_PERSONAL_ACCESS_TOKENS_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapPersonalAccessTokens(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task<RawPersonalAccessToken?> CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null,
- CancellationToken token = default)
- {
- var message = TcpContracts.CreatePersonalAccessToken(name, DurationHelpers.ToDuration(expiry));
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE);
-
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapRawPersonalAccessToken(responseBuffer.Memory.Span);
- }
-
- /// <inheritdoc />
- public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default)
- {
- var message = TcpContracts.DeletePersonalRequestToken(name);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE);
-
- await SendAckAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<AuthResponse?> LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default)
- {
- var message = TcpContracts.LoginWithPersonalAccessToken(token);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE);
-
- SetConnectionStateAsync(ConnectionState.Authenticating);
- using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, ct);
-
- if (responseBuffer.Memory.Length == 0)
- {
- return null;
- }
-
- var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..4]);
-
- SetConnectionStateAsync(ConnectionState.Authenticated);
-
- if (await RedirectAsync(ct))
- {
- await ConnectAsync(ct);
- return await LoginWithPersonalAccessTokenAsync(token, ct);
- }
-
- return new AuthResponse(userId, null);
- }
-
- private async Task TryEstablishConnectionAsync(CancellationToken token)
- {
- var retryCount = 0;
- var delay = _configuration.ReconnectionSettings.InitialDelay;
- do
- {
- // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
- _stream?.Close();
- _stream?.Dispose();
-
- if (string.IsNullOrEmpty(_currentAddress))
- {
- _currentAddress = _configuration.BaseAddress;
- }
-
- var urlPortSplitter = _currentAddress.Split(":");
- if (urlPortSplitter.Length > 2)
- {
- throw new InvalidBaseAddressException();
- }
-
- try
- {
- var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
- socket.SendBufferSize = _configuration.SendBufferSize;
- socket.ReceiveBufferSize = _configuration.ReceiveBufferSize;
-
- await socket.ConnectAsync(urlPortSplitter[0], int.Parse(urlPortSplitter[1]), token);
-
- socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true);
- socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5);
-
- SetConnectionStateAsync(ConnectionState.Connected);
- _lastConnectionTime = DateTimeOffset.UtcNow;
-
- _stream = _configuration.TlsSettings.Enabled switch
- {
- true => await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings),
- false => new TcpConnectionStream(new NetworkStream(socket, true))
- };
-
- if (_configuration.AutoLoginSettings.Enabled)
- {
- _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}",
- _configuration.AutoLoginSettings.Username);
- await LoginUserAsync(_configuration.AutoLoginSettings.Username,
- _configuration.AutoLoginSettings.Password, token);
- }
-
- break;
- }
- catch (Exception e)
- {
- _logger.LogError(e, "Failed to connect");
-
- if (!_configuration.ReconnectionSettings.Enabled ||
- (_configuration.ReconnectionSettings.MaxRetries > 0 &&
- retryCount >= _configuration.ReconnectionSettings.MaxRetries))
- {
- SetConnectionStateAsync(ConnectionState.Disconnected);
- throw;
- }
-
- retryCount++;
- if (_configuration.ReconnectionSettings.UseExponentialBackoff)
- {
- delay *= _configuration.ReconnectionSettings.BackoffMultiplier;
-
- if (delay > _configuration.ReconnectionSettings.MaxDelay)
- {
- delay = _configuration.ReconnectionSettings.MaxDelay;
- }
- }
-
- if (_logger.IsEnabled(LogLevel.Information))
- {
- _logger.LogInformation("Retrying connection attempt {RetryCount} with delay {Delay}", retryCount,
- delay);
- }
-
- await Task.Delay(delay, token);
- }
- } while (true);
- }
-
- private async Task<ClusterNode?> GetCurrentLeaderNodeAsync(CancellationToken token)
- {
- try
- {
- var clusterMetadata = await GetClusterMetadataAsync(token);
- if (clusterMetadata == null)
- {
- return null;
- }
-
- // Single-node cluster (clustering disabled) - no redirection needed
- if (clusterMetadata.Nodes.Count() == 1)
- {
- return null;
- }
-
- var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => x.Role == ClusterNodeRole.Leader);
- if (leaderNode == null)
- {
- throw new MissingLeaderException();
- }
-
- return leaderNode;
- }
- // todo: change after error refactoring, error code 5 is for feature not supported
- catch (IggyInvalidStatusCodeException e) when (e.StatusCode == 5)
- {
- return null;
- }
- }
-
- private async Task<TcpConnectionStream> CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings)
- {
- ValidateCertificatePath(tlsSettings.CertificatePath);
-
- _customCaStore = new X509Certificate2Collection();
- _customCaStore.ImportFromPemFile(tlsSettings.CertificatePath);
- var stream = new NetworkStream(socket, true);
- var sslStream = new SslStream(stream, false, RemoteCertificateValidationCallback);
-
- await sslStream.AuthenticateAsClientAsync(tlsSettings.Hostname);
-
- return new TcpConnectionStream(sslStream);
- }
-
- private async Task SendAckAsync(ReadOnlyMemory<byte> payload, CancellationToken token = default)
- {
- using IMemoryOwner<byte> _ = await SendWithResponseAsync(payload, token);
- }
-
- private async Task<IMemoryOwner<byte>> SendWithResponseAsync(ReadOnlyMemory<byte> payload,
- CancellationToken token = default)
- {
- try
- {
- return await SendRawAsync(payload, token);
- }
- catch (Exception e) when (IsConnectionException(e) && !_isConnecting)
- {
- _logger.LogWarning("Connection lost");
- if (!_configuration.ReconnectionSettings.Enabled)
- {
- _logger.LogWarning("Reconnection is disabled");
- SetConnectionStateAsync(ConnectionState.Disconnected);
- throw;
- }
-
- return await HandleReconnectionAsync(payload, token);
- }
- }
-
- private async Task<IMemoryOwner<byte>> HandleReconnectionAsync(ReadOnlyMemory<byte> payload,
- CancellationToken token)
- {
- var currentTime = DateTimeOffset.UtcNow;
- await _connectionSemaphore.WaitAsync(token);
-
- try
- {
- if (_state is ConnectionState.Connected or ConnectionState.Authenticated
- && _lastConnectionTime > currentTime)
- {
- _logger.LogInformation("Connection already established, sending payload");
- return await SendRawAsync(payload, token);
- }
-
- SetConnectionStateAsync(ConnectionState.Disconnected);
- _logger.LogInformation("Reconnecting to the server");
- await ConnectAsync(token);
-
- _logger.LogInformation("Reconnected to the server");
-
- await Task.Delay(_configuration.ReconnectionSettings.WaitAfterReconnect, token);
-
- return await SendRawAsync(payload, token);
- }
- finally
- {
- _connectionSemaphore.Release();
- }
- }
-
- private async Task<IMemoryOwner<byte>> SendRawAsync(ReadOnlyMemory<byte> payload, CancellationToken token)
- {
- if (_state is ConnectionState.Disconnected or ConnectionState.Connecting)
- {
- throw new NotConnectedException();
- }
-
- await _sendingSemaphore.WaitAsync(token);
-
- try
- {
- await _stream.SendAsync(payload, token);
- await _stream.FlushAsync(token);
-
- // Read the 8-byte header (4 bytes status + 4 bytes length)
- var totalRead = 0;
- while (totalRead < BufferSizes.EXPECTED_RESPONSE_SIZE)
- {
- var readBytes
- = await _stream.ReadAsync(
- _responseHeaderBuffer.AsMemory(totalRead, BufferSizes.EXPECTED_RESPONSE_SIZE - totalRead),
- token);
- if (readBytes == 0)
- {
- throw new IggyZeroBytesException();
- }
-
- totalRead += readBytes;
- }
-
- var response = TcpMessageStreamHelpers.GetResponseLengthAndStatus(_responseHeaderBuffer);
-
- if (response.Status != 0)
- {
- if (response.Length == 0)
- {
- throw new IggyInvalidStatusCodeException(response.Status,
- $"Invalid response status code: {response.Status}");
- }
-
- var errorBuffer = new byte[response.Length];
- totalRead = 0;
- while (totalRead < response.Length)
- {
- var readBytes
- = await _stream.ReadAsync(errorBuffer.AsMemory(totalRead, response.Length - totalRead), token);
- if (readBytes == 0)
- {
- throw new IggyZeroBytesException();
- }
-
- totalRead += readBytes;
- }
-
- throw new InvalidResponseException(Encoding.UTF8.GetString(errorBuffer));
- }
-
- if (response.Length == 0)
- {
- return EmptyMemoryOwner.Instance;
- }
-
- var responseBuffer = ArrayPoolHelper.Rent(response.Length);
- try
- {
- totalRead = 0;
- while (totalRead < response.Length)
- {
- var readBytes
- = await _stream.ReadAsync(responseBuffer.Memory.Slice(totalRead, response.Length - totalRead),
- token);
-
- if (readBytes == 0)
- {
- throw new IggyZeroBytesException();
- }
-
- totalRead += readBytes;
- }
- }
- catch
- {
- responseBuffer.Dispose();
- throw;
- }
-
- return responseBuffer;
- }
- finally
- {
- _sendingSemaphore.Release();
- }
- }
-
- private static bool IsConnectionException(Exception ex)
- {
- return ex is IggyZeroBytesException or
- NotConnectedException or
- SocketException or
- IOException or
- ObjectDisposedException;
- }
-
- private static int CalculatePayloadBufferSize(int messageBufferSize)
- {
- return messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH;
- }
-
- private static int CalculateMessageBufferSize(Identifier streamId, Identifier topicId, Consumer consumer)
- {
- // Original: 14 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.Id.Length
- // Added 1 byte for partition flag
- return 15 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.ConsumerId.Length;
- }
-
- /// <summary>
- /// Sets the connection state and fires the OnConnectionStateChanged event.
- /// </summary>
- /// <param name="newState">The new connection state</param>
- private void SetConnectionStateAsync(ConnectionState newState)
- {
- if (_state == newState)
- {
- return;
- }
-
- var previousState = _state;
- _state = newState;
-
- _logger.LogInformation("Connection state changed: {PreviousState} -> {CurrentState}", previousState, newState);
- _connectionEvents.Publish(new ConnectionStateChangedEventArgs(previousState, newState));
- }
-
- private void ValidateCertificatePath(string tlsCertificatePath)
- {
- if (string.IsNullOrEmpty(tlsCertificatePath)
- || !File.Exists(tlsCertificatePath))
- {
- throw new InvalidCertificatePathException(tlsCertificatePath);
- }
- }
-
- private bool RemoteCertificateValidationCallback(object sender, X509Certificate? certificate, X509Chain? chain,
- SslPolicyErrors sslPolicyErrors)
- {
- if (sslPolicyErrors == SslPolicyErrors.None)
- {
- return true;
- }
-
- if (certificate is null)
- {
- return false;
- }
-
- if (certificate is not X509Certificate2 serverCert)
- {
- serverCert = new X509Certificate2(certificate);
- }
-
- if (_customCaStore.Any(ca => ca.Thumbprint == serverCert.Thumbprint))
- {
- if (DateTime.UtcNow <= serverCert.NotAfter && DateTime.UtcNow >= serverCert.NotBefore)
- {
- return true;
- }
-
- _logger.LogError(
- "Server certificate matches trusted key but is expired. Valid from {NotBefore} to {NotAfter}",
- serverCert.NotBefore, serverCert.NotAfter);
- return false;
- }
-
-
- using var customChain = new X509Chain();
- customChain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
- customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
- foreach (var ca in _customCaStore)
- {
- customChain.ChainPolicy.CustomTrustStore.Add(ca);
- customChain.ChainPolicy.ExtraStore.Add(ca);
- }
-
- customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
-
- if (customChain.Build(new X509Certificate2(certificate)))
- {
- if (!sslPolicyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch))
- {
- return true;
- }
-
- _logger.LogError("Custom CA chain is valid, but hostname does not match");
- return false;
- }
-
- foreach (var chainStatus in customChain.ChainStatus)
- {
- _logger.LogWarning("Certificate validation failed: {ChainStatus} - {StatusInformation}", chainStatus.Status,
- chainStatus.StatusInformation);
- }
-
- return false;
- }
-
- private async Task<bool> RedirectAsync(CancellationToken token)
- {
- var currentLeaderNode = await GetCurrentLeaderNodeAsync(token);
- if (currentLeaderNode == null)
- {
- return false;
- }
-
- var leaderAddress = $"{currentLeaderNode.Ip}:{currentLeaderNode.Endpoints.Tcp}";
- if (leaderAddress == _currentAddress)
- {
- return false;
- }
-
- _currentAddress = leaderAddress;
-
- _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}",
- leaderAddress);
-
- _stream.Close();
- SetConnectionStateAsync(ConnectionState.Disconnected);
- return true;
- }
-
- internal sealed class EmptyMemoryOwner : IMemoryOwner<byte>
- {
- public static readonly EmptyMemoryOwner Instance = new();
-
- private EmptyMemoryOwner()
- {
- }
-
- public Memory<byte> Memory => Memory<byte>.Empty;
-
- public void Dispose()
- {
- }
- }
-}
-
-internal static class ArrayPoolHelper
-{
- public static SlicedMemoryOwner Rent(int minimumLength)
- {
- return new SlicedMemoryOwner(minimumLength);
- }
-
- internal sealed class SlicedMemoryOwner(int minimumLength) : IMemoryOwner<byte>
- {
- private readonly byte[] _value = ArrayPool<byte>.Shared.Rent(minimumLength);
- private int _disposed;
-
- public void Dispose()
- {
- if (Interlocked.Exchange(ref _disposed, 1) != 0)
- {
- return;
- }
-
- ArrayPool<byte>.Shared.Return(_value);
- }
-
- public Memory<byte> Memory => _value.AsMemory()[..minimumLength];
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Buffers;
+using System.Buffers.Binary;
+using System.Net.Security;
+using System.Net.Sockets;
+using System.Security.Cryptography.X509Certificates;
+using System.Text;
+using Apache.Iggy.Configuration;
+using Apache.Iggy.ConnectionStream;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Contracts.Auth;
+using Apache.Iggy.Contracts.Tcp;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Mappers;
+using Apache.Iggy.Messages;
+using Apache.Iggy.Utils;
+using Microsoft.Extensions.Logging;
+using Partitioning = Apache.Iggy.Kinds.Partitioning;
+
+namespace Apache.Iggy.IggyClient.Implementations;
+
+/// <summary>
+/// A TCP client for interacting with the Iggy server.
+/// </summary>
+public sealed class TcpMessageStream : IIggyClient
+{
+ private readonly IggyClientConfigurator _configuration;
+ private readonly EventAggregator<ConnectionStateChangedEventArgs> _connectionEvents;
+ private readonly SemaphoreSlim _connectionSemaphore;
+ private readonly ILogger<TcpMessageStream> _logger;
+ private readonly byte[] _responseHeaderBuffer = new byte[BufferSizes.EXPECTED_RESPONSE_SIZE];
+ private readonly SemaphoreSlim _sendingSemaphore;
+ private string _currentAddress = string.Empty;
+ private X509Certificate2Collection _customCaStore = [];
+ private bool _isConnecting;
+ private DateTimeOffset _lastConnectionTime;
+ private ConnectionState _state = ConnectionState.Disconnected;
+ private TcpConnectionStream _stream = null!;
+
+ internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory loggerFactory)
+ {
+ _configuration = configuration;
+ _logger = loggerFactory.CreateLogger<TcpMessageStream>();
+ _sendingSemaphore = new SemaphoreSlim(1, 1);
+ _connectionSemaphore = new SemaphoreSlim(1, 1);
+ _lastConnectionTime = DateTimeOffset.MinValue;
+ _connectionEvents = new EventAggregator<ConnectionStateChangedEventArgs>(loggerFactory);
+ }
+
+ /// <summary>
+ /// Fired whenever the connection state changes.
+ /// </summary>
+ //public event EventHandler<ConnectionStateChangedEventArgs>? OnConnectionStateChanged;
+ public void Dispose()
+ {
+ _stream?.Close();
+ _stream?.Dispose();
+ _sendingSemaphore.Dispose();
+ _connectionSemaphore.Dispose();
+ _connectionEvents.Clear();
+ }
+
+ /// <inheritdoc />
+ public void SubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
+ {
+ _connectionEvents.Subscribe(callback);
+ }
+
+ /// <inheritdoc />
+ public void UnsubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
+ {
+ _connectionEvents.Unsubscribe(callback);
+ }
+
+ /// <inheritdoc />
+ public string GetCurrentAddress()
+ {
+ return _currentAddress;
+ }
+
+ /// <inheritdoc />
+ public async Task<StreamResponse?> CreateStreamAsync(string name, CancellationToken token = default)
+ {
+ var message = TcpContracts.CreateStream(name);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_STREAM_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ throw new InvalidResponseException("Received empty response while trying to create stream.");
+ }
+
+ return BinaryMapper.MapStream(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<StreamResponse?> GetStreamByIdAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAM_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapStream(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<StreamResponse>> GetStreamsAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAMS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapStreams(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default)
+ {
+ var message = TcpContracts.UpdateStream(streamId, name);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_STREAM_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_STREAM_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_STREAM_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<TopicResponse>> GetTopicsAsync(Identifier streamId,
+ CancellationToken token = default)
+ {
+ var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPICS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapTopics(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<TopicResponse?> GetTopicByIdAsync(Identifier streamId, Identifier topicId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.GetTopicById(streamId, topicId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPIC_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapTopic(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<TopicResponse?> CreateTopicAsync(Identifier streamId, string name, uint partitionsCount,
+ CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null,
+ TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default)
+ {
+ var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry);
+ var message = TcpContracts.CreateTopic(streamId, name, partitionsCount, compressionAlgorithm,
+ replicationFactor, messageExpiryValue, maxTopicSize);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_TOPIC_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapTopic(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name,
+ CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None,
+ ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null,
+ CancellationToken token = default)
+ {
+ var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry);
+ var message = TcpContracts.UpdateTopic(streamId, topicId, name, compressionAlgorithm, maxTopicSize,
+ messageExpiryValue, replicationFactor);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_TOPIC_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
+ {
+ var message = TcpContracts.DeleteTopic(streamId, topicId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_TOPIC_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
+ {
+ var message = TcpContracts.PurgeTopic(streamId, topicId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_TOPIC_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+
+ /// <inheritdoc />
+ public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
+ IList<Message> messages, CancellationToken token = default)
+ {
+ var metadataLength = 2 + streamId.Length + 2 + topicId.Length
+ + 2 + partitioning.Length + 4 + 4;
+ var messageBufferSize = TcpMessageStreamHelpers.CalculateMessageBytesCount(messages)
+ + metadataLength;
+ var payloadBufferSize = messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH;
+
+ IMemoryOwner<byte> messageBuffer = MemoryPool<byte>.Shared.Rent(messageBufferSize);
+ IMemoryOwner<byte> payloadBuffer = MemoryPool<byte>.Shared.Rent(payloadBufferSize);
+ try
+ {
+ TcpContracts.CreateMessage(messageBuffer.Memory.Span[..messageBufferSize], streamId,
+ topicId, partitioning, messages);
+
+ TcpMessageStreamHelpers.CreatePayload(payloadBuffer.Memory.Span[..payloadBufferSize],
+ messageBuffer.Memory.Span[..messageBufferSize], CommandCodes.SEND_MESSAGES_CODE);
+
+ await SendAckAsync(payloadBuffer.Memory[..payloadBufferSize], token);
+ }
+ finally
+ {
+ messageBuffer.Dispose();
+ payloadBuffer.Dispose();
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.FlushUnsavedBuffer(streamId, topicId, partitionId, fsync);
+
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.FLUSH_UNSAVED_BUFFER_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
+ Consumer consumer,
+ PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
+ {
+ using var rental = await PollMessagesRentedAsync(streamId, topicId, partitionId, consumer, pollingStrategy,
+ count, autoCommit, token);
+ return BinaryMapper.MaterializeMessages(rental);
+ }
+
+ /// <inheritdoc />
+ public async Task<PolledMessagesRental> PollMessagesRentedAsync(Identifier streamId, Identifier topicId,
+ uint? partitionId,
+ Consumer consumer,
+ PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
+ {
+ var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer);
+ var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize);
+ var payload = ArrayPool<byte>.Shared.Rent(payloadBufferSize);
+
+ try
+ {
+ TcpContracts.GetMessages(payload.AsSpan().Slice(8, messageBufferSize), consumer, streamId,
+ topicId, pollingStrategy, count, autoCommit, partitionId);
+ BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[..4], messageBufferSize + 4);
+ BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[4..8], CommandCodes.POLL_MESSAGES_CODE);
+
+ IMemoryOwner<byte> responseBuffer
+ = await SendWithResponseAsync(payload.AsMemory(0, payloadBufferSize), token);
+ return BinaryMapper.MapRentedMessages(responseBuffer);
+ }
+ finally
+ {
+ ArrayPool<byte>.Shared.Return(payload);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset,
+ uint? partitionId, CancellationToken token = default)
+ {
+ var message = TcpContracts.UpdateOffset(streamId, topicId, consumer, offset, partitionId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.STORE_CONSUMER_OFFSET_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<OffsetResponse?> GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId,
+ uint? partitionId, CancellationToken token = default)
+ {
+ var message = TcpContracts.GetOffset(streamId, topicId, consumer, partitionId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_OFFSET_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapOffsets(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.DeleteOffset(streamId, topicId, consumer, partitionId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_OFFSET_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<ConsumerGroupResponse>> GetConsumerGroupsAsync(Identifier streamId,
+ Identifier topicId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.GetGroups(streamId, topicId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUPS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapConsumerGroups(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<ConsumerGroupResponse?> GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId,
+ Identifier groupId, CancellationToken token = default)
+ {
+ var message = TcpContracts.GetGroup(streamId, topicId, groupId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUP_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<ConsumerGroupResponse?> CreateConsumerGroupAsync(Identifier streamId, Identifier topicId,
+ string name, CancellationToken token = default)
+ {
+ var message = TcpContracts.CreateGroup(streamId, topicId, name);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_CONSUMER_GROUP_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.DeleteGroup(streamId, topicId, groupId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_GROUP_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.JoinGroup(streamId, topicId, groupId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.JOIN_CONSUMER_GROUP_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.LeaveGroup(streamId, topicId, groupId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LEAVE_CONSUMER_GROUP_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.DeletePartitions(streamId, topicId, partitionsCount);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PARTITIONS_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.CreatePartitions(streamId, topicId, partitionsCount);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PARTITIONS_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId,
+ uint segmentsCount, CancellationToken token = default)
+ {
+ var message = TcpContracts.DeleteSegments(streamId, topicId, partitionId, segmentsCount);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_SEGMENTS_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<ClientResponse?> GetMeAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_ME_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapClient(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<StatsResponse?> GetStatsAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STATS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapStats(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<ClusterMetadata?> GetClusterMetadataAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLUSTER_METADATA_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapClusterMetadata(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task PingAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PING_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<byte[]> GetSnapshotAsync(SnapshotCompression compression,
+ IList<SystemSnapshotType> snapshotTypes, CancellationToken token = default)
+ {
+ var message = TcpContracts.GetSnapshot(compression, snapshotTypes);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_SNAPSHOT_CODE);
+
+ using IMemoryOwner<byte> result = await SendWithResponseAsync(payload, token);
+
+ return result.Memory.Span.ToArray();
+ }
+
+ /// <inheritdoc />
+ public async Task ConnectAsync(CancellationToken token = default)
+ {
+ if (_state is ConnectionState.Connected
+ or ConnectionState.Authenticating
+ or ConnectionState.Authenticated)
+ {
+ _logger.LogWarning("Connection is already connected");
+ return;
+ }
+
+ if (_lastConnectionTime != DateTimeOffset.MinValue)
+ {
+ await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token);
+ }
+
+ SetConnectionStateAsync(ConnectionState.Connecting);
+ _isConnecting = true;
+ try
+ {
+ await TryEstablishConnectionAsync(token);
+ }
+ finally
+ {
+ _isConnecting = false;
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<ClientResponse>> GetClientsAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENTS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapClients(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<ClientResponse?> GetClientByIdAsync(uint clientId, CancellationToken token = default)
+ {
+ var message = TcpContracts.GetClient(clientId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENT_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapClient(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<UserResponse?> GetUserAsync(Identifier userId, CancellationToken token = default)
+ {
+ var message = TcpContracts.GetUser(userId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USER_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapUser(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<UserResponse>> GetUsersAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USERS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapUsers(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<UserResponse?> CreateUserAsync(string userName, string password, UserStatus status,
+ Permissions? permissions = null, CancellationToken token = default)
+ {
+ var message = TcpContracts.CreateUser(userName, password, status, permissions);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_USER_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapUser(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default)
+ {
+ var message = TcpContracts.DeleteUser(userId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_USER_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.UpdateUser(userId, userName, status);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_USER_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.UpdatePermissions(userId, permissions);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_PERMISSIONS_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.ChangePassword(userId, currentPassword, newPassword);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CHANGE_PASSWORD_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<AuthResponse?> LoginUserAsync(string userName, string password, CancellationToken token = default)
+ {
+ if (_state == ConnectionState.Disconnected)
+ {
+ throw new NotConnectedException();
+ }
+
+ // TODO: Add binary protocol version
+ var message = TcpContracts.LoginUser(userName, password, SdkVersion.Value, "csharp-sdk");
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_USER_CODE);
+
+ SetConnectionStateAsync(ConnectionState.Authenticating);
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..responseBuffer.Memory.Length]);
+ SetConnectionStateAsync(ConnectionState.Authenticated);
+
+ if (await RedirectAsync(token))
+ {
+ await ConnectAsync(token);
+ return await LoginUserAsync(userName, password, token);
+ }
+
+ var authResponse = new AuthResponse(userId, null);
+ return authResponse;
+ }
+
+ /// <inheritdoc />
+ public async Task LogoutUserAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGOUT_USER_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<PersonalAccessTokenResponse>> GetPersonalAccessTokensAsync(
+ CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_PERSONAL_ACCESS_TOKENS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapPersonalAccessTokens(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<RawPersonalAccessToken?> CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.CreatePersonalAccessToken(name, DurationHelpers.ToDuration(expiry));
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapRawPersonalAccessToken(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default)
+ {
+ var message = TcpContracts.DeletePersonalRequestToken(name);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<AuthResponse?> LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default)
+ {
+ var message = TcpContracts.LoginWithPersonalAccessToken(token);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE);
+
+ SetConnectionStateAsync(ConnectionState.Authenticating);
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, ct);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..4]);
+
+ SetConnectionStateAsync(ConnectionState.Authenticated);
+
+ if (await RedirectAsync(ct))
+ {
+ await ConnectAsync(ct);
+ return await LoginWithPersonalAccessTokenAsync(token, ct);
+ }
+
+ return new AuthResponse(userId, null);
+ }
+
+ private async Task TryEstablishConnectionAsync(CancellationToken token)
+ {
+ var retryCount = 0;
+ var delay = _configuration.ReconnectionSettings.InitialDelay;
+ do
+ {
+ // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
+ _stream?.Close();
+ _stream?.Dispose();
+
+ if (string.IsNullOrEmpty(_currentAddress))
+ {
+ _currentAddress = _configuration.BaseAddress;
+ }
+
+ var urlPortSplitter = _currentAddress.Split(":");
+ if (urlPortSplitter.Length > 2)
+ {
+ throw new InvalidBaseAddressException();
+ }
+
+ try
+ {
+ var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+ socket.SendBufferSize = _configuration.SendBufferSize;
+ socket.ReceiveBufferSize = _configuration.ReceiveBufferSize;
+
+ await socket.ConnectAsync(urlPortSplitter[0], int.Parse(urlPortSplitter[1]), token);
+
+ socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true);
+ socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5);
+
+ SetConnectionStateAsync(ConnectionState.Connected);
+ _lastConnectionTime = DateTimeOffset.UtcNow;
+
+ _stream = _configuration.TlsSettings.Enabled switch
+ {
+ true => await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings),
+ false => new TcpConnectionStream(new NetworkStream(socket, true))
+ };
+
+ if (_configuration.AutoLoginSettings.Enabled)
+ {
+ _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}",
+ _configuration.AutoLoginSettings.Username);
+ await LoginUserAsync(_configuration.AutoLoginSettings.Username,
+ _configuration.AutoLoginSettings.Password, token);
+ }
+
+ break;
+ }
+ catch (Exception e)
+ {
+ _logger.LogError(e, "Failed to connect");
+
+ if (!_configuration.ReconnectionSettings.Enabled ||
+ (_configuration.ReconnectionSettings.MaxRetries > 0 &&
+ retryCount >= _configuration.ReconnectionSettings.MaxRetries))
+ {
+ SetConnectionStateAsync(ConnectionState.Disconnected);
+ throw;
+ }
+
+ retryCount++;
+ if (_configuration.ReconnectionSettings.UseExponentialBackoff)
+ {
+ delay *= _configuration.ReconnectionSettings.BackoffMultiplier;
+
+ if (delay > _configuration.ReconnectionSettings.MaxDelay)
+ {
+ delay = _configuration.ReconnectionSettings.MaxDelay;
+ }
+ }
+
+ if (_logger.IsEnabled(LogLevel.Information))
+ {
+ _logger.LogInformation("Retrying connection attempt {RetryCount} with delay {Delay}", retryCount,
+ delay);
+ }
+
+ await Task.Delay(delay, token);
+ }
+ } while (true);
+ }
+
+ private async Task<ClusterNode?> GetCurrentLeaderNodeAsync(CancellationToken token)
+ {
+ try
+ {
+ var clusterMetadata = await GetClusterMetadataAsync(token);
+ if (clusterMetadata == null)
+ {
+ return null;
+ }
+
+ // Single-node cluster (clustering disabled) - no redirection needed
+ if (clusterMetadata.Nodes.Count() == 1)
+ {
+ return null;
+ }
+
+ var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => x.Role == ClusterNodeRole.Leader);
+ if (leaderNode == null)
+ {
+ throw new MissingLeaderException();
+ }
+
+ return leaderNode;
+ }
+ // todo: change after error refactoring, error code 5 is for feature not supported
+ catch (IggyInvalidStatusCodeException e) when (e.StatusCode == 5)
+ {
+ return null;
+ }
+ }
+
+ private async Task<TcpConnectionStream> CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings)
+ {
+ ValidateCertificatePath(tlsSettings.CertificatePath);
+
+ _customCaStore = new X509Certificate2Collection();
+ _customCaStore.ImportFromPemFile(tlsSettings.CertificatePath);
+ var stream = new NetworkStream(socket, true);
+ var sslStream = new SslStream(stream, false, RemoteCertificateValidationCallback);
+
+ await sslStream.AuthenticateAsClientAsync(tlsSettings.Hostname);
+
+ return new TcpConnectionStream(sslStream);
+ }
+
+ private async Task SendAckAsync(ReadOnlyMemory<byte> payload, CancellationToken token = default)
+ {
+ using IMemoryOwner<byte> _ = await SendWithResponseAsync(payload, token);
+ }
+
+ private async Task<IMemoryOwner<byte>> SendWithResponseAsync(ReadOnlyMemory<byte> payload,
+ CancellationToken token = default)
+ {
+ try
+ {
+ return await SendRawAsync(payload, token);
+ }
+ catch (Exception e) when (IsConnectionException(e) && !_isConnecting)
+ {
+ _logger.LogWarning("Connection lost");
+ if (!_configuration.ReconnectionSettings.Enabled)
+ {
+ _logger.LogWarning("Reconnection is disabled");
+ SetConnectionStateAsync(ConnectionState.Disconnected);
+ throw;
+ }
+
+ return await HandleReconnectionAsync(payload, token);
+ }
+ }
+
+ private async Task<IMemoryOwner<byte>> HandleReconnectionAsync(ReadOnlyMemory<byte> payload,
+ CancellationToken token)
+ {
+ var currentTime = DateTimeOffset.UtcNow;
+ await _connectionSemaphore.WaitAsync(token);
+
+ try
+ {
+ if (_state is ConnectionState.Connected or ConnectionState.Authenticated
+ && _lastConnectionTime > currentTime)
+ {
+ _logger.LogInformation("Connection already established, sending payload");
+ return await SendRawAsync(payload, token);
+ }
+
+ SetConnectionStateAsync(ConnectionState.Disconnected);
+ _logger.LogInformation("Reconnecting to the server");
+ await ConnectAsync(token);
+
+ _logger.LogInformation("Reconnected to the server");
+
+ await Task.Delay(_configuration.ReconnectionSettings.WaitAfterReconnect, token);
+
+ return await SendRawAsync(payload, token);
+ }
+ finally
+ {
+ _connectionSemaphore.Release();
+ }
+ }
+
+ private async Task<IMemoryOwner<byte>> SendRawAsync(ReadOnlyMemory<byte> payload, CancellationToken token)
+ {
+ if (_state is ConnectionState.Disconnected or ConnectionState.Connecting)
+ {
+ throw new NotConnectedException();
+ }
+
+ await _sendingSemaphore.WaitAsync(token);
+
+ try
+ {
+ await _stream.SendAsync(payload, token);
+ await _stream.FlushAsync(token);
+
+ // Read the 8-byte header (4 bytes status + 4 bytes length)
+ var totalRead = 0;
+ while (totalRead < BufferSizes.EXPECTED_RESPONSE_SIZE)
+ {
+ var readBytes
+ = await _stream.ReadAsync(
+ _responseHeaderBuffer.AsMemory(totalRead, BufferSizes.EXPECTED_RESPONSE_SIZE - totalRead),
+ token);
+ if (readBytes == 0)
+ {
+ throw new IggyZeroBytesException();
+ }
+
+ totalRead += readBytes;
+ }
+
+ var response = TcpMessageStreamHelpers.GetResponseLengthAndStatus(_responseHeaderBuffer);
+
+ if (response.Status != 0)
+ {
+ if (response.Length == 0)
+ {
+ throw new IggyInvalidStatusCodeException(response.Status,
+ $"Invalid response status code: {response.Status}");
+ }
+
+ var errorBuffer = new byte[response.Length];
+ totalRead = 0;
+ while (totalRead < response.Length)
+ {
+ var readBytes
+ = await _stream.ReadAsync(errorBuffer.AsMemory(totalRead, response.Length - totalRead), token);
+ if (readBytes == 0)
+ {
+ throw new IggyZeroBytesException();
+ }
+
+ totalRead += readBytes;
+ }
+
+ throw new InvalidResponseException(Encoding.UTF8.GetString(errorBuffer));
+ }
+
+ if (response.Length == 0)
+ {
+ return EmptyMemoryOwner.Instance;
+ }
+
+ var responseBuffer = ArrayPoolHelper.Rent(response.Length);
+ try
+ {
+ totalRead = 0;
+ while (totalRead < response.Length)
+ {
+ var readBytes
+ = await _stream.ReadAsync(responseBuffer.Memory.Slice(totalRead, response.Length - totalRead),
+ token);
+
+ if (readBytes == 0)
+ {
+ throw new IggyZeroBytesException();
+ }
+
+ totalRead += readBytes;
+ }
+ }
+ catch
+ {
+ responseBuffer.Dispose();
+ throw;
+ }
+
+ return responseBuffer;
+ }
+ finally
+ {
+ _sendingSemaphore.Release();
+ }
+ }
+
+ private static bool IsConnectionException(Exception ex)
+ {
+ return ex is IggyZeroBytesException or
+ NotConnectedException or
+ SocketException or
+ IOException or
+ ObjectDisposedException;
+ }
+
+ private static int CalculatePayloadBufferSize(int messageBufferSize)
+ {
+ return messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH;
+ }
+
+ private static int CalculateMessageBufferSize(Identifier streamId, Identifier topicId, Consumer consumer)
+ {
+ // Original: 14 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.Id.Length
+ // Added 1 byte for partition flag
+ return 15 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.ConsumerId.Length;
+ }
+
+ /// <summary>
+ /// Sets the connection state and fires the OnConnectionStateChanged event.
+ /// </summary>
+ /// <param name="newState">The new connection state</param>
+ private void SetConnectionStateAsync(ConnectionState newState)
+ {
+ if (_state == newState)
+ {
+ return;
+ }
+
+ var previousState = _state;
+ _state = newState;
+
+ _logger.LogInformation("Connection state changed: {PreviousState} -> {CurrentState}", previousState, newState);
+ _connectionEvents.Publish(new ConnectionStateChangedEventArgs(previousState, newState));
+ }
+
+ private void ValidateCertificatePath(string tlsCertificatePath)
+ {
+ if (string.IsNullOrEmpty(tlsCertificatePath)
+ || !File.Exists(tlsCertificatePath))
+ {
+ throw new InvalidCertificatePathException(tlsCertificatePath);
+ }
+ }
+
+ private bool RemoteCertificateValidationCallback(object sender, X509Certificate? certificate, X509Chain? chain,
+ SslPolicyErrors sslPolicyErrors)
+ {
+ if (sslPolicyErrors == SslPolicyErrors.None)
+ {
+ return true;
+ }
+
+ if (certificate is null)
+ {
+ return false;
+ }
+
+ if (certificate is not X509Certificate2 serverCert)
+ {
+ serverCert = new X509Certificate2(certificate);
+ }
+
+ if (_customCaStore.Any(ca => ca.Thumbprint == serverCert.Thumbprint))
+ {
+ if (DateTime.UtcNow <= serverCert.NotAfter && DateTime.UtcNow >= serverCert.NotBefore)
+ {
+ return true;
+ }
+
+ _logger.LogError(
+ "Server certificate matches trusted key but is expired. Valid from {NotBefore} to {NotAfter}",
+ serverCert.NotBefore, serverCert.NotAfter);
+ return false;
+ }
+
+
+ using var customChain = new X509Chain();
+ customChain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
+ customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
+ foreach (var ca in _customCaStore)
+ {
+ customChain.ChainPolicy.CustomTrustStore.Add(ca);
+ customChain.ChainPolicy.ExtraStore.Add(ca);
+ }
+
+ customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
+
+ if (customChain.Build(new X509Certificate2(certificate)))
+ {
+ if (!sslPolicyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch))
+ {
+ return true;
+ }
+
+ _logger.LogError("Custom CA chain is valid, but hostname does not match");
+ return false;
+ }
+
+ foreach (var chainStatus in customChain.ChainStatus)
+ {
+ _logger.LogWarning("Certificate validation failed: {ChainStatus} - {StatusInformation}", chainStatus.Status,
+ chainStatus.StatusInformation);
+ }
+
+ return false;
+ }
+
+ private async Task<bool> RedirectAsync(CancellationToken token)
+ {
+ var currentLeaderNode = await GetCurrentLeaderNodeAsync(token);
+ if (currentLeaderNode == null)
+ {
+ return false;
+ }
+
+ var leaderAddress = $"{currentLeaderNode.Ip}:{currentLeaderNode.Endpoints.Tcp}";
+ if (leaderAddress == _currentAddress)
+ {
+ return false;
+ }
+
+ _currentAddress = leaderAddress;
+
+ _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}",
+ leaderAddress);
+
+ _stream.Close();
+ SetConnectionStateAsync(ConnectionState.Disconnected);
+ return true;
+ }
+
+ internal sealed class EmptyMemoryOwner : IMemoryOwner<byte>
+ {
+ public static readonly EmptyMemoryOwner Instance = new();
+
+ private EmptyMemoryOwner()
+ {
+ }
+
+ public Memory<byte> Memory => Memory<byte>.Empty;
+
+ public void Dispose()
+ {
+ }
+ }
+}
+
+internal static class ArrayPoolHelper
+{
+ public static SlicedMemoryOwner Rent(int minimumLength)
+ {
+ return new SlicedMemoryOwner(minimumLength);
+ }
+
+ internal sealed class SlicedMemoryOwner(int minimumLength) : IMemoryOwner<byte>
+ {
+ private readonly byte[] _value = ArrayPool<byte>.Shared.Rent(minimumLength);
+ private int _disposed;
+
+ public void Dispose()
+ {
+ if (Interlocked.Exchange(ref _disposed, 1) != 0)
+ {
+ return;
+ }
+
+ ArrayPool<byte>.Shared.Return(_value);
+ }
+
+ public Memory<byte> Memory => _value.AsMemory()[..minimumLength];
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs b/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs
index ffdbf3260..dc41363dc 100644
--- a/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs
+++ b/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs
@@ -1,106 +1,106 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Text.Json;
-using System.Text.Json.Serialization;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Headers;
-using Apache.Iggy.Messages;
-
-namespace Apache.Iggy.JsonConverters;
-
-internal sealed class MessageResponseConverter : JsonConverter<MessageResponse>
-{
- private static readonly UserHeadersConverter HeadersConverter = new();
-
- public override MessageResponse Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
- {
- if (reader.TokenType != JsonTokenType.StartObject)
- {
- throw new JsonException("Expected start of object for MessageResponse.");
- }
-
- MessageHeader? header = null;
- byte[]? payload = null;
- Dictionary<HeaderKey, HeaderValue>? userHeaders = null;
- byte[]? rawUserHeaders = null;
-
- while (reader.Read())
- {
- if (reader.TokenType == JsonTokenType.EndObject)
- {
- break;
- }
-
- if (reader.TokenType != JsonTokenType.PropertyName)
- {
- throw new JsonException("Expected property name.");
- }
-
- var propertyName = reader.GetString();
- reader.Read();
-
- switch (propertyName)
- {
- case "header":
- header = JsonSerializer.Deserialize<MessageHeader>(ref reader, options);
- break;
- case "payload":
- payload = reader.GetBytesFromBase64();
- break;
- case "user_headers":
- if (reader.TokenType == JsonTokenType.String)
- {
- rawUserHeaders = reader.GetBytesFromBase64();
- }
- else if (reader.TokenType == JsonTokenType.Null)
- {
- userHeaders = null;
- }
- else
- {
- userHeaders = HeadersConverter.Read(ref reader, typeof(Dictionary<HeaderKey, HeaderValue>),
- options);
- }
-
- break;
- default:
- reader.Skip();
- break;
- }
- }
-
- var response = new MessageResponse
- {
- Header = header ?? throw new JsonException("Missing 'header' field."),
- Payload = payload ?? throw new JsonException("Missing 'payload' field."),
- RawUserHeaders = rawUserHeaders
- };
-
- if (rawUserHeaders is null)
- {
- response.UserHeaders = userHeaders;
- }
-
- return response;
- }
-
- public override void Write(Utf8JsonWriter writer, MessageResponse value, JsonSerializerOptions options)
- {
- JsonSerializer.Serialize(writer, value, options);
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Headers;
+using Apache.Iggy.Messages;
+
+namespace Apache.Iggy.JsonConverters;
+
+internal sealed class MessageResponseConverter : JsonConverter<MessageResponse>
+{
+ private static readonly UserHeadersConverter HeadersConverter = new();
+
+ public override MessageResponse Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+ {
+ if (reader.TokenType != JsonTokenType.StartObject)
+ {
+ throw new JsonException("Expected start of object for MessageResponse.");
+ }
+
+ MessageHeader? header = null;
+ byte[]? payload = null;
+ Dictionary<HeaderKey, HeaderValue>? userHeaders = null;
+ byte[]? rawUserHeaders = null;
+
+ while (reader.Read())
+ {
+ if (reader.TokenType == JsonTokenType.EndObject)
+ {
+ break;
+ }
+
+ if (reader.TokenType != JsonTokenType.PropertyName)
+ {
+ throw new JsonException("Expected property name.");
+ }
+
+ var propertyName = reader.GetString();
+ reader.Read();
+
+ switch (propertyName)
+ {
+ case "header":
+ header = JsonSerializer.Deserialize<MessageHeader>(ref reader, options);
+ break;
+ case "payload":
+ payload = reader.GetBytesFromBase64();
+ break;
+ case "user_headers":
+ if (reader.TokenType == JsonTokenType.String)
+ {
+ rawUserHeaders = reader.GetBytesFromBase64();
+ }
+ else if (reader.TokenType == JsonTokenType.Null)
+ {
+ userHeaders = null;
+ }
+ else
+ {
+ userHeaders = HeadersConverter.Read(ref reader, typeof(Dictionary<HeaderKey, HeaderValue>),
+ options);
+ }
+
+ break;
+ default:
+ reader.Skip();
+ break;
+ }
+ }
+
+ var response = new MessageResponse
+ {
+ Header = header ?? throw new JsonException("Missing 'header' field."),
+ Payload = payload ?? throw new JsonException("Missing 'payload' field."),
+ RawUserHeaders = rawUserHeaders
+ };
+
+ if (rawUserHeaders is null)
+ {
+ response.UserHeaders = userHeaders;
+ }
+
+ return response;
+ }
+
+ public override void Write(Utf8JsonWriter writer, MessageResponse value, JsonSerializerOptions options)
+ {
+ JsonSerializer.Serialize(writer, value, options);
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
index ad47f0399..390a63624 100644
--- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
+++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
@@ -228,38 +228,34 @@ internal static class BinaryMapper
var readBytes = 4 + 8 + 1 + 1 + usernameLength;
return (new UserResponse
- {
- Id = id,
- CreatedAt = createdAt,
- Status = userStatus,
- Username = username
- },
+ {
+ Id = id,
+ CreatedAt = createdAt,
+ Status = userStatus,
+ Username = username
+ },
readBytes);
}
internal static ClientResponse MapClient(ReadOnlySpan<byte> payload)
{
var (response, position) = MapClientInfo(payload, 0);
- var consumerGroups = new List<ConsumerGroupInfo>();
- var length = payload.Length;
+ var consumerGroups = new List<ConsumerGroupInfo>(response.ConsumerGroupsCount);
- while (position < length)
+ for (var i = 0; i < response.ConsumerGroupsCount; i++)
{
- for (var i = 0; i < response.ConsumerGroupsCount; i++)
- {
- var streamId = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]);
- var topicId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 4)..(position + 8)]);
- var consumerGroupId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 8)..(position + 12)]);
- var consumerGroup
- = new ConsumerGroupInfo
- {
- StreamId = streamId,
- TopicId = topicId,
- GroupId = consumerGroupId
- };
- consumerGroups.Add(consumerGroup);
- position += 12;
- }
+ var streamId = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]);
+ var topicId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 4)..(position + 8)]);
+ var consumerGroupId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 8)..(position + 12)]);
+ var consumerGroup
+ = new ConsumerGroupInfo
+ {
+ StreamId = streamId,
+ TopicId = topicId,
+ GroupId = consumerGroupId
+ };
+ consumerGroups.Add(consumerGroup);
+ position += 12;
}
return new ClientResponse
@@ -350,7 +346,7 @@ internal static class BinaryMapper
}
}
- internal static PolledMessagesRental MapRentedMessages(ReadOnlyMemory<byte> payload,
+ private static PolledMessagesRental MapRentedMessages(ReadOnlyMemory<byte> payload,
IMemoryOwner<byte> payloadOwner)
{
ReadOnlySpan<byte> span = payload.Span;
@@ -953,11 +949,11 @@ internal static class BinaryMapper
}
return (new ConsumerGroupMember
- {
- Id = id,
- PartitionsCount = partitionsCount,
- Partitions = partitions
- },
+ {
+ Id = id,
+ PartitionsCount = partitionsCount,
+ Partitions = partitions
+ },
8 + partitionsCount * 4);
}
diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs
index e9a8084a8..1644bdc9c 100644
--- a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs
+++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs
@@ -1,483 +1,483 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Buffers;
-using System.Runtime.CompilerServices;
-using System.Text;
-using Apache.Iggy.Consumers;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Encryption;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.IggyClient;
-using Apache.Iggy.IggyClient.Implementations;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Messages;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
-using Moq;
-
-namespace Apache.Iggy.Tests.ConsumerTests;
-
-public class RentedConsumerTests
-{
- [Fact]
- public async Task ReceiveRentedAsync_Should_YieldMessages_WithExpectedPayloads()
- {
- var owner = new TrackingMemoryOwner(1024);
- IReadOnlyList<RentedMessageResponse> messages = BuildMessages(owner, 3);
- var rental = new PolledMessagesRental(owner)
- {
- PartitionId = 1,
- CurrentOffset = 2,
- Messages = messages
- };
- Mock<IIggyClient> client = BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
- var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance);
- await consumer.InitAsync(TestContext.Current.CancellationToken);
-
- var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
- var got = new List<ReceivedRentedMessage>();
- await using (IAsyncEnumerator<ReceivedRentedMessage> e = consumer.ReceiveRentedAsync(cts.Token)
- .GetAsyncEnumerator(TestContext.Current.CancellationToken))
- {
- for (var i = 0; i < 3; i++)
- {
- Assert.True(await e.MoveNextAsync());
- got.Add(e.Current);
- }
- }
-
- Assert.Equal(3, got.Count);
- for (var i = 0; i < 3; i++)
- {
- Assert.Equal(MessageStatus.Success, got[i].Status);
- Assert.Equal($"msg-{i}", Encoding.UTF8.GetString(got[i].Message.Payload.Span));
- Assert.Equal((ulong)i, got[i].CurrentOffset);
- Assert.Equal(1u, got[i].PartitionId);
- Assert.Null(got[i].Error);
- }
-
- // Buffer still rented — none of the messages have been disposed.
- Assert.Equal(0, owner.DisposeCount);
-
- foreach (var m in got)
- {
- m.Dispose();
- }
-
- // After disposing every message of the batch, rental returned to pool exactly once.
- Assert.Equal(1, owner.DisposeCount);
- await consumer.DisposeAsync();
- }
-
- [Fact]
- public async Task ReceiveRentedAsync_Should_NotReturnBuffer_UntilAllMessagesDisposed()
- {
- var owner = new TrackingMemoryOwner(1024);
- IReadOnlyList<RentedMessageResponse> messages = BuildMessages(owner, 3);
- var rental = new PolledMessagesRental(owner)
- {
- PartitionId = 1,
- CurrentOffset = 2,
- Messages = messages
- };
- Mock<IIggyClient> client = BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
- var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance);
- await consumer.InitAsync(TestContext.Current.CancellationToken);
-
- var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
- var got = new List<ReceivedRentedMessage>();
- await using (IAsyncEnumerator<ReceivedRentedMessage> e = consumer.ReceiveRentedAsync(cts.Token)
- .GetAsyncEnumerator(TestContext.Current.CancellationToken))
- {
- for (var i = 0; i < 3; i++)
- {
- Assert.True(await e.MoveNextAsync());
- got.Add(e.Current);
- }
- }
-
- // Dispose only the first two: buffer must still be alive.
- got[0].Dispose();
- Assert.Equal(0, owner.DisposeCount);
- got[1].Dispose();
- Assert.Equal(0, owner.DisposeCount);
-
- // Final dispose returns the buffer.
- got[2].Dispose();
- Assert.Equal(1, owner.DisposeCount);
-
- // Calling Dispose again is a no-op — refcount must not drop below zero.
- got[0].Dispose();
- got[1].Dispose();
- got[2].Dispose();
- Assert.Equal(1, owner.DisposeCount);
-
- await consumer.DisposeAsync();
- }
-
- [Fact]
- public async Task ReceiveRentedAsync_Should_YieldDecryptionFailed_AndStillReleaseBuffer_WhenEncryptorThrows()
- {
- var owner = new TrackingMemoryOwner(1024);
- IReadOnlyList<RentedMessageResponse> messages = BuildMessages(owner, 1);
- var rental = new PolledMessagesRental(owner)
- {
- PartitionId = 1,
- CurrentOffset = 0,
- Messages = messages
- };
- Mock<IIggyClient> client = BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
- var encryptor = new ThrowingEncryptor();
- var config = BuildConfig();
- config.MessageEncryptor = encryptor;
- var consumer = new IggyConsumer(client.Object, config, NullLoggerFactory.Instance);
- await consumer.InitAsync(TestContext.Current.CancellationToken);
-
- var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
- ReceivedRentedMessage? got = null;
- await using (IAsyncEnumerator<ReceivedRentedMessage> e = consumer.ReceiveRentedAsync(cts.Token)
- .GetAsyncEnumerator(TestContext.Current.CancellationToken))
- {
- Assert.True(await e.MoveNextAsync());
- got = e.Current;
- }
-
- Assert.NotNull(got);
- Assert.Equal(MessageStatus.DecryptionFailed, got!.Status);
- Assert.IsType<InvalidOperationException>(got.Error);
-
- Assert.Equal(0, owner.DisposeCount);
- got.Dispose();
- Assert.Equal(1, owner.DisposeCount);
-
- await consumer.DisposeAsync();
- }
-
- [Fact]
- public async Task ReceiveRentedAsync_Should_Throw_WhenConsumerNotInitialized()
- {
- Mock<IIggyClient> client = BuildClientMock(new Queue<PolledMessagesRental>());
- var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance);
-
- await Assert.ThrowsAsync<ConsumerNotInitializedException>(async () =>
- {
- await foreach (var _ in consumer.ReceiveRentedAsync(TestContext.Current.CancellationToken))
- {
- }
- });
-
- await consumer.DisposeAsync();
- }
-
- [Fact]
- public void RentedBatchHandle_Release_BeyondAcquired_Throws()
- {
- var owner = new TrackingMemoryOwner(16);
- var rental = new PolledMessagesRental(owner)
- {
- PartitionId = 1,
- CurrentOffset = 0,
- Messages = Array.Empty<RentedMessageResponse>()
- };
- var handle = new RentedBatchHandle(rental);
-
- handle.Release(); // releases self-ref -> 0 -> disposes rental
- Assert.Equal(1, owner.DisposeCount);
-
- Assert.Throws<InvalidOperationException>(() => handle.Release());
- }
-
- [Fact]
- public void RentedBatchHandle_AcquireRelease_Balanced_DisposesOnce()
- {
- var owner = new TrackingMemoryOwner(16);
- var rental = new PolledMessagesRental(owner)
- {
- PartitionId = 1,
- CurrentOffset = 0,
- Messages = Array.Empty<RentedMessageResponse>()
- };
- var handle = new RentedBatchHandle(rental); // refCount=1 (self-ref)
-
- handle.Acquire(); // 2
- handle.Acquire(); // 3
- Assert.Equal(0, owner.DisposeCount);
-
- handle.Release(); // 2
- handle.Release(); // 1 (self-ref still alive)
- Assert.Equal(0, owner.DisposeCount);
-
- handle.Release(); // 0 -> disposed
- Assert.Equal(1, owner.DisposeCount);
- }
-
- [Fact]
- public void PolledMessagesRental_Dispose_Idempotent()
- {
- var owner = new TrackingMemoryOwner(16);
- var rental = new PolledMessagesRental(owner)
- {
- PartitionId = 1,
- CurrentOffset = 0,
- Messages = Array.Empty<RentedMessageResponse>()
- };
-
- rental.Dispose();
- rental.Dispose();
- rental.Dispose();
-
- Assert.Equal(1, owner.DisposeCount);
- }
-
- [Fact]
- public async Task PolledMessagesRental_ConcurrentDispose_ReturnsBufferOnce()
- {
- var owner = new TrackingMemoryOwner(16);
- var rental = new PolledMessagesRental(owner)
- {
- PartitionId = 1,
- CurrentOffset = 0,
- Messages = Array.Empty<RentedMessageResponse>()
- };
-
- using var start = new ManualResetEventSlim(false);
- Task[] tasks = Enumerable.Range(0, 64).Select(_ => Task.Run(() =>
- {
- start.Wait();
- rental.Dispose();
- })).ToArray();
-
- start.Set();
- await Task.WhenAll(tasks);
-
- Assert.Equal(1, owner.DisposeCount);
- }
-
- [Fact]
- public void PolledMessagesRental_ForgotDispose_FinalizerReturnsBuffer()
- {
- var owner = new TrackingMemoryOwner(16);
- MakeAndDrop(owner);
-
- GC.Collect();
- GC.WaitForPendingFinalizers();
- GC.Collect();
-
- Assert.Equal(1, owner.DisposeCount);
-
- [MethodImpl(MethodImplOptions.NoInlining)]
- static void MakeAndDrop(IMemoryOwner<byte> o)
- {
- _ = new PolledMessagesRental(o)
- {
- PartitionId = 1,
- CurrentOffset = 0,
- Messages = Array.Empty<RentedMessageResponse>()
- };
- }
- }
-
- [Fact]
- public async Task PollRented_MidLoopPublishFailure_DoesNotLeakBuffer()
- {
- var owner = new TrackingMemoryOwner(1024);
- IReadOnlyList<RentedMessageResponse> messages = BuildMessages(owner, 5);
- var rental = new PolledMessagesRental(owner)
- {
- PartitionId = 1,
- CurrentOffset = 4,
- Messages = messages
- };
- Mock<IIggyClient> client = BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
- var consumer
- = new FailingPublishConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance) { FailAfter = 2 };
- await consumer.InitAsync(TestContext.Current.CancellationToken);
-
- using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
- var got = new List<ReceivedRentedMessage>();
- IAsyncEnumerator<ReceivedRentedMessage> enumerator = consumer.ReceiveRentedAsync(cts.Token)
- .GetAsyncEnumerator(TestContext.Current.CancellationToken);
-
- for (var i = 0; i < 2; i++)
- {
- Assert.True(await enumerator.MoveNextAsync());
- got.Add(enumerator.Current);
- }
-
- // First 2 publishes succeeded, 3rd injected failure aborted the loop.
- // Producer self-ref was released in finally; consumer refs (2) still hold the buffer.
- Assert.Equal(2, got.Count);
- Assert.Equal(0, owner.DisposeCount);
-
- foreach (var m in got)
- {
- m.Dispose();
- }
-
- // All refs released -> rental disposed exactly once.
- Assert.Equal(1, owner.DisposeCount);
-
- cts.Cancel();
- try
- {
- await enumerator.DisposeAsync();
- }
- catch (OperationCanceledException)
- {
- }
-
- await consumer.DisposeAsync();
- }
-
- internal static IggyConsumerConfig BuildConfig()
- {
- return new IggyConsumerConfig
- {
- StreamId = Identifier.Numeric(1),
- TopicId = Identifier.Numeric(1),
- Consumer = Consumer.New(1),
- PollingStrategy = PollingStrategy.Next(),
- BatchSize = 10,
- PartitionId = 1,
- AutoCommitMode = AutoCommitMode.Disabled,
- AutoCommit = false,
- PollingIntervalMs = 0
- };
- }
-
- /// <summary>
- /// Slices payload bytes into the supplied owner's memory and returns a list of
- /// <see cref="RentedMessageResponse" /> instances backed by that single rented buffer.
- /// </summary>
- internal static IReadOnlyList<RentedMessageResponse> BuildMessages(TrackingMemoryOwner owner, int count)
- {
- var list = new List<RentedMessageResponse>(count);
- Memory<byte> buffer = owner.Memory;
- var written = 0;
- for (var i = 0; i < count; i++)
- {
- var bytes = Encoding.UTF8.GetBytes($"msg-{i}");
- bytes.CopyTo(buffer.Slice(written, bytes.Length));
- Memory<byte> slice = buffer.Slice(written, bytes.Length);
- written += bytes.Length;
-
- list.Add(new RentedMessageResponse
- {
- Header = new MessageHeader
- {
- Offset = (ulong)i,
- PayloadLength = bytes.Length
- },
- Payload = slice,
- RawUserHeaders = ReadOnlyMemory<byte>.Empty
- });
- }
-
- return list;
- }
-
- /// <summary>
- /// Builds a <see cref="Mock{IIggyClient}" /> that dequeues rentals on each
- /// <c>PollMessagesRentedAsync</c> call. When the queue is empty, returns an
- /// empty rental so the consumer can spin without dereferencing null.
- /// </summary>
- internal static Mock<IIggyClient> BuildClientMock(Queue<PolledMessagesRental> rentals)
- {
- var mock = new Mock<IIggyClient>(MockBehavior.Loose);
- mock.Setup(c => c.ConnectAsync(It.IsAny<CancellationToken>())).Returns(Task.CompletedTask);
- mock.Setup(c => c.PollMessagesRentedAsync(It.IsAny<Identifier>(), It.IsAny<Identifier>(), It.IsAny<uint?>(),
- It.IsAny<Consumer>(), It.IsAny<PollingStrategy>(), It.IsAny<uint>(), It.IsAny<bool>(),
- It.IsAny<CancellationToken>()))
- .ReturnsAsync(() =>
- {
- if (rentals.Count > 0)
- {
- return rentals.Dequeue();
- }
-
- return new PolledMessagesRental(TcpMessageStream.EmptyMemoryOwner.Instance)
- {
- PartitionId = 1,
- CurrentOffset = 0,
- Messages = Array.Empty<RentedMessageResponse>()
- };
- });
- return mock;
- }
-
- /// <summary>
- /// <see cref="IMemoryOwner{T}" /> wrapper that counts how many times
- /// <see cref="IDisposable.Dispose" /> has been invoked, so tests can assert that
- /// the rental returns to the pool exactly once.
- /// </summary>
- internal sealed class TrackingMemoryOwner : IMemoryOwner<byte>
- {
- private readonly byte[] _buffer;
-
- public int DisposeCount { get; private set; }
-
- public TrackingMemoryOwner(int size)
- {
- _buffer = new byte[size];
- }
-
- public Memory<byte> Memory => _buffer;
-
- public void Dispose()
- {
- DisposeCount++;
- }
- }
-
- private sealed class ThrowingEncryptor : IMessageEncryptor
- {
- public byte[] Encrypt(Span<byte> plainData)
- {
- throw new NotSupportedException();
- }
-
- public byte[] Decrypt(ReadOnlySpan<byte> encryptedData)
- {
- throw new InvalidOperationException("decrypt fail");
- }
- }
-
- private sealed class FailingPublishConsumer : IggyConsumer
- {
- private int _calls;
-
- public int FailAfter { get; set; }
-
- public FailingPublishConsumer(IIggyClient client, IggyConsumerConfig config,
- ILoggerFactory loggerFactory)
- : base(client, config, loggerFactory)
- {
- }
-
- protected override async Task PublishRentedAsync(RentedBatchHandle rental, RentedMessageResponse message,
- uint partitionId, MessageStatus status, Exception? error, CancellationToken ct)
- {
- if (Interlocked.Increment(ref _calls) > FailAfter)
- {
- throw new InvalidOperationException("inject publish failure");
- }
-
- await base.PublishRentedAsync(rental, message, partitionId, status, error, ct);
- }
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
... 854 lines suppressed ...