You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by br...@apache.org on 2024/11/11 21:46:12 UTC
(arrow) 02/15: GH-44361: [C#][Integration] Include .NET in Flight integration tests (#44377)
This is an automated email from the ASF dual-hosted git repository.
brycemecum pushed a commit to branch maint-18.1.0
in repository https://gitbox.apache.org/repos/asf/arrow.git
commit f42c5b0c240d7275ec28081aa9be71f17a61e3bf
Author: Adam Reeve <ad...@gmail.com>
AuthorDate: Wed Oct 16 03:09:41 2024 +1300
GH-44361: [C#][Integration] Include .NET in Flight integration tests (#44377)
### Rationale for this change
See #44361. This allows testing compatibility of the .NET Flight implementation with other Flight implementations.
### What changes are included in this PR?
* Adds a new `Apache.Arrow.Flight.IntegrationTest` project that can run in server or client mode for Flight integration tests.
* Includes the integration tests that send then retrieve data defined in JSON files, but doesn't add any of the named scenarios
* Configures archery to include C# in the Flight integration tests, but skip all the named scenarios
* Also skips tests that use dictionary data due to #38045, and the empty data test due to #44363
### Are these changes tested?
These changes are tests.
### Are there any user-facing changes?
No
* GitHub Issue: #44361
Authored-by: Adam Reeve <ad...@gmail.com>
Signed-off-by: Curt Hagenlocher <cu...@hagenlocher.org>
---
csharp/Apache.Arrow.sln | 6 +
.../FlightRecordBatchStreamWriter.cs | 2 +-
.../Apache.Arrow.Flight.IntegrationTest.csproj | 18 +++
.../FlightClientCommand.cs | 51 +++++++
.../FlightServerCommand.cs | 68 +++++++++
.../GrpcResolver.cs | 34 +++++
.../JsonTestScenario.cs | 167 +++++++++++++++++++++
.../Apache.Arrow.Flight.IntegrationTest/Program.cs | 67 +++++++++
csharp/test/Apache.Arrow.Flight.TestWeb/Startup.cs | 11 +-
.../TestFlightServer.cs | 8 +-
dev/archery/archery/integration/datagen.py | 23 ++-
dev/archery/archery/integration/runner.py | 9 +-
dev/archery/archery/integration/tester_csharp.py | 57 ++++++-
dev/release/rat_exclude_files.txt | 1 +
14 files changed, 502 insertions(+), 20 deletions(-)
diff --git a/csharp/Apache.Arrow.sln b/csharp/Apache.Arrow.sln
index 7e7f7c6331..0e569de1d6 100644
--- a/csharp/Apache.Arrow.sln
+++ b/csharp/Apache.Arrow.sln
@@ -27,6 +27,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Flight.Sql.Tes
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Flight.Sql", "src\Apache.Arrow.Flight.Sql\Apache.Arrow.Flight.Sql.csproj", "{2ADE087A-B424-4895-8CC5-10170D10BA62}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Flight.IntegrationTest", "test\Apache.Arrow.Flight.IntegrationTest\Apache.Arrow.Flight.IntegrationTest.csproj", "{7E66CBB4-D921-41E7-A98A-7C6DEA521696}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -81,6 +83,10 @@ Global
{2ADE087A-B424-4895-8CC5-10170D10BA62}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2ADE087A-B424-4895-8CC5-10170D10BA62}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2ADE087A-B424-4895-8CC5-10170D10BA62}.Release|Any CPU.Build.0 = Release|Any CPU
+ {7E66CBB4-D921-41E7-A98A-7C6DEA521696}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {7E66CBB4-D921-41E7-A98A-7C6DEA521696}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {7E66CBB4-D921-41E7-A98A-7C6DEA521696}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {7E66CBB4-D921-41E7-A98A-7C6DEA521696}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
diff --git a/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamWriter.cs b/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamWriter.cs
index f76f082245..7a8a6fd677 100644
--- a/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamWriter.cs
+++ b/csharp/src/Apache.Arrow.Flight/FlightRecordBatchStreamWriter.cs
@@ -64,7 +64,7 @@ namespace Apache.Arrow.Flight
{
if (!_disposed)
{
- _flightDataStream.Dispose();
+ _flightDataStream?.Dispose();
_disposed = true;
}
}
diff --git a/csharp/test/Apache.Arrow.Flight.IntegrationTest/Apache.Arrow.Flight.IntegrationTest.csproj b/csharp/test/Apache.Arrow.Flight.IntegrationTest/Apache.Arrow.Flight.IntegrationTest.csproj
new file mode 100644
index 0000000000..34030621b4
--- /dev/null
+++ b/csharp/test/Apache.Arrow.Flight.IntegrationTest/Apache.Arrow.Flight.IntegrationTest.csproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <OutputType>Exe</OutputType>
+ <TargetFramework>net8.0</TargetFramework>
+ <RootNamespace>Apache.Arrow.Flight.IntegrationTest</RootNamespace>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+ <PackageReference Include="System.Text.Json" Version="8.0.5" />
+ <ProjectReference Include="..\..\src\Apache.Arrow.Flight\Apache.Arrow.Flight.csproj" />
+ <ProjectReference Include="..\Apache.Arrow.Flight.TestWeb\Apache.Arrow.Flight.TestWeb.csproj" />
+ <ProjectReference Include="..\Apache.Arrow.IntegrationTest\Apache.Arrow.IntegrationTest.csproj" />
+ </ItemGroup>
+
+</Project>
diff --git a/csharp/test/Apache.Arrow.Flight.IntegrationTest/FlightClientCommand.cs b/csharp/test/Apache.Arrow.Flight.IntegrationTest/FlightClientCommand.cs
new file mode 100644
index 0000000000..d9e0ff5230
--- /dev/null
+++ b/csharp/test/Apache.Arrow.Flight.IntegrationTest/FlightClientCommand.cs
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Apache.Arrow.Flight.IntegrationTest;
+
+public class FlightClientCommand
+{
+ private readonly int _port;
+ private readonly string _scenario;
+ private readonly FileInfo _jsonFileInfo;
+
+ public FlightClientCommand(int port, string scenario, FileInfo jsonFileInfo)
+ {
+ _port = port;
+ _scenario = scenario;
+ _jsonFileInfo = jsonFileInfo;
+ }
+
+ public async Task Execute()
+ {
+ if (!string.IsNullOrEmpty(_scenario))
+ {
+ // No named scenarios are currently implemented
+ throw new Exception($"Scenario '{_scenario}' is not supported.");
+ }
+
+ if (!(_jsonFileInfo?.Exists ?? false))
+ {
+ throw new Exception($"Invalid JSON file path '{_jsonFileInfo?.FullName}'");
+ }
+
+ var scenario = new JsonTestScenario(_port, _jsonFileInfo);
+ await scenario.RunClient().ConfigureAwait(false);
+ }
+}
diff --git a/csharp/test/Apache.Arrow.Flight.IntegrationTest/FlightServerCommand.cs b/csharp/test/Apache.Arrow.Flight.IntegrationTest/FlightServerCommand.cs
new file mode 100644
index 0000000000..c3a7694485
--- /dev/null
+++ b/csharp/test/Apache.Arrow.Flight.IntegrationTest/FlightServerCommand.cs
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Net;
+using System.Threading.Tasks;
+using Apache.Arrow.Flight.TestWeb;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Hosting.Server;
+using Microsoft.AspNetCore.Hosting.Server.Features;
+using Microsoft.AspNetCore.Server.Kestrel.Core;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+
+namespace Apache.Arrow.Flight.IntegrationTest;
+
+public class FlightServerCommand
+{
+ private readonly string _scenario;
+
+ public FlightServerCommand(string scenario)
+ {
+ _scenario = scenario;
+ }
+
+ public async Task Execute()
+ {
+ if (!string.IsNullOrEmpty(_scenario))
+ {
+ // No named scenarios are currently implemented
+ throw new Exception($"Scenario '{_scenario}' is not supported.");
+ }
+
+ var host = Host.CreateDefaultBuilder()
+ .ConfigureWebHostDefaults(webBuilder =>
+ {
+ webBuilder
+ .ConfigureKestrel(options =>
+ {
+ options.Listen(IPEndPoint.Parse("127.0.0.1:0"), l => l.Protocols = HttpProtocols.Http2);
+ })
+ .UseStartup<Startup>();
+ })
+ .Build();
+
+ await host.StartAsync().ConfigureAwait(false);
+
+ var addresses = host.Services.GetService<IServer>().Features.Get<IServerAddressesFeature>().Addresses;
+ foreach (var address in addresses)
+ {
+ Console.WriteLine($"Server listening on {address}");
+ }
+
+ await host.WaitForShutdownAsync().ConfigureAwait(false);
+ }
+}
diff --git a/csharp/test/Apache.Arrow.Flight.IntegrationTest/GrpcResolver.cs b/csharp/test/Apache.Arrow.Flight.IntegrationTest/GrpcResolver.cs
new file mode 100644
index 0000000000..44b1075e7a
--- /dev/null
+++ b/csharp/test/Apache.Arrow.Flight.IntegrationTest/GrpcResolver.cs
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Grpc.Net.Client.Balancer;
+
+namespace Apache.Arrow.Flight.IntegrationTest;
+
+/// <summary>
+/// The Grpc.Net.Client library doesn't know how to handle the "grpc+tcp" scheme used by Arrow Flight.
+/// This ResolverFactory passes these through to the standard Static Resolver used for the http scheme.
+/// </summary>
+public class GrpcTcpResolverFactory : ResolverFactory
+{
+ public override string Name => "grpc+tcp";
+
+ public override Resolver Create(ResolverOptions options)
+ {
+ return new StaticResolverFactory(
+ uri => new[] { new BalancerAddress(options.Address.Host, options.Address.Port) })
+ .Create(options);
+ }
+}
diff --git a/csharp/test/Apache.Arrow.Flight.IntegrationTest/JsonTestScenario.cs b/csharp/test/Apache.Arrow.Flight.IntegrationTest/JsonTestScenario.cs
new file mode 100644
index 0000000000..f4f3ac28bf
--- /dev/null
+++ b/csharp/test/Apache.Arrow.Flight.IntegrationTest/JsonTestScenario.cs
@@ -0,0 +1,167 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Apache.Arrow.Flight.Client;
+using Apache.Arrow.IntegrationTest;
+using Apache.Arrow.Tests;
+using Apache.Arrow.Types;
+using Google.Protobuf;
+using Grpc.Net.Client;
+using Grpc.Core;
+using Grpc.Net.Client.Balancer;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Apache.Arrow.Flight.IntegrationTest;
+
+/// <summary>
+/// A test scenario defined using a JSON data file
+/// </summary>
+internal class JsonTestScenario
+{
+ private readonly int _serverPort;
+ private readonly FileInfo _jsonFile;
+ private readonly ServiceProvider _serviceProvider;
+
+ public JsonTestScenario(int serverPort, FileInfo jsonFile)
+ {
+ _serverPort = serverPort;
+ _jsonFile = jsonFile;
+
+ var services = new ServiceCollection();
+ services.AddSingleton<ResolverFactory>(new GrpcTcpResolverFactory());
+ _serviceProvider = services.BuildServiceProvider();
+ }
+
+ public async Task RunClient()
+ {
+ var address = $"grpc+tcp://localhost:{_serverPort}";
+ using var channel = GrpcChannel.ForAddress(
+ address,
+ new GrpcChannelOptions
+ {
+ ServiceProvider = _serviceProvider,
+ Credentials = ChannelCredentials.Insecure
+ });
+ var client = new FlightClient(channel);
+
+ var descriptor = FlightDescriptor.CreatePathDescriptor(_jsonFile.FullName);
+
+ var jsonFile = await JsonFile.ParseAsync(_jsonFile).ConfigureAwait(false);
+ var schema = jsonFile.GetSchemaAndDictionaries(out Func<DictionaryType, IArrowArray> dictionaries);
+ var batches = jsonFile.Batches.Select(batch => batch.ToArrow(schema, dictionaries)).ToArray();
+
+ // 1. Put the data to the server.
+ await UploadBatches(client, descriptor, batches).ConfigureAwait(false);
+
+ // 2. Get the ticket for the data.
+ var info = await client.GetInfo(descriptor).ConfigureAwait(false);
+ if (info.Endpoints.Count == 0)
+ {
+ throw new Exception("No endpoints received");
+ }
+
+ // 3. Stream data from the server, comparing individual batches.
+ foreach (var endpoint in info.Endpoints)
+ {
+ var locations = endpoint.Locations.ToArray();
+ if (locations.Length == 0)
+ {
+ // Can read with existing client
+ await ConsumeFlightLocation(client, endpoint.Ticket, batches).ConfigureAwait(false);
+ }
+ else
+ {
+ foreach (var location in locations)
+ {
+ using var readChannel = GrpcChannel.ForAddress(
+ location.Uri,
+ new GrpcChannelOptions
+ {
+ ServiceProvider = _serviceProvider,
+ Credentials = ChannelCredentials.Insecure
+ });
+ var readClient = new FlightClient(readChannel);
+ await ConsumeFlightLocation(readClient, endpoint.Ticket, batches).ConfigureAwait(false);
+ }
+ }
+ }
+ }
+
+ private static async Task UploadBatches(FlightClient client, FlightDescriptor descriptor, RecordBatch[] batches)
+ {
+ using var putCall = client.StartPut(descriptor);
+ using var writer = putCall.RequestStream;
+
+ try
+ {
+ var counter = 0;
+ foreach (var batch in batches)
+ {
+ var metadata = $"{counter}";
+
+ await writer.WriteAsync(batch, ByteString.CopyFromUtf8(metadata)).ConfigureAwait(false);
+
+ // Verify server has acknowledged the write request
+ await putCall.ResponseStream.MoveNext().ConfigureAwait(false);
+ var responseString = putCall.ResponseStream.Current.ApplicationMetadata.ToStringUtf8();
+
+ if (responseString != metadata)
+ {
+ throw new Exception($"Response metadata '{responseString}' does not match expected metadata '{metadata}'");
+ }
+
+ counter++;
+ }
+ }
+ finally
+ {
+ await writer.CompleteAsync().ConfigureAwait(false);
+ }
+
+ // Drain the response stream to ensure the server has stored the data
+ var hasMore = await putCall.ResponseStream.MoveNext().ConfigureAwait(false);
+ if (hasMore)
+ {
+ throw new Exception("Expected to have reached the end of the response stream");
+ }
+ }
+
+ private static async Task ConsumeFlightLocation(FlightClient client, FlightTicket ticket, RecordBatch[] batches)
+ {
+ using var readStream = client.GetStream(ticket);
+ var counter = 0;
+ foreach (var originalBatch in batches)
+ {
+ if (!await readStream.ResponseStream.MoveNext().ConfigureAwait(false))
+ {
+ throw new Exception($"Expected {batches.Length} batches but received {counter}");
+ }
+
+ var batch = readStream.ResponseStream.Current;
+ ArrowReaderVerifier.CompareBatches(originalBatch, batch, strictCompare: false);
+
+ counter++;
+ }
+
+ if (await readStream.ResponseStream.MoveNext().ConfigureAwait(false))
+ {
+ throw new Exception($"Expected to reach the end of the response stream after {batches.Length} batches");
+ }
+ }
+}
diff --git a/csharp/test/Apache.Arrow.Flight.IntegrationTest/Program.cs b/csharp/test/Apache.Arrow.Flight.IntegrationTest/Program.cs
new file mode 100644
index 0000000000..24d39de28a
--- /dev/null
+++ b/csharp/test/Apache.Arrow.Flight.IntegrationTest/Program.cs
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System.CommandLine;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Apache.Arrow.Flight.IntegrationTest;
+
+public static class Program
+{
+ public static async Task<int> Main(string[] args)
+ {
+ var portOption = new Option<int>(
+ new[] { "--port", "-p" },
+ description: "Port the Flight server is listening on");
+ var scenarioOption = new Option<string>(
+ new[] { "--scenario", "-s" },
+ "The name of the scenario to run");
+ var pathOption = new Option<FileInfo>(
+ new[] { "--path", "-j" },
+ "Path to a JSON file of test data");
+
+ var rootCommand = new RootCommand(
+ "Integration test application for Apache.Arrow .NET Flight.");
+
+ var clientCommand = new Command("client", "Run the Flight client")
+ {
+ portOption,
+ scenarioOption,
+ pathOption,
+ };
+ rootCommand.AddCommand(clientCommand);
+
+ clientCommand.SetHandler(async (port, scenario, jsonFile) =>
+ {
+ var command = new FlightClientCommand(port, scenario, jsonFile);
+ await command.Execute().ConfigureAwait(false);
+ }, portOption, scenarioOption, pathOption);
+
+ var serverCommand = new Command("server", "Run the Flight server")
+ {
+ scenarioOption,
+ };
+ rootCommand.AddCommand(serverCommand);
+
+ serverCommand.SetHandler(async scenario =>
+ {
+ var command = new FlightServerCommand(scenario);
+ await command.Execute().ConfigureAwait(false);
+ }, scenarioOption);
+
+ return await rootCommand.InvokeAsync(args).ConfigureAwait(false);
+ }
+}
diff --git a/csharp/test/Apache.Arrow.Flight.TestWeb/Startup.cs b/csharp/test/Apache.Arrow.Flight.TestWeb/Startup.cs
index 97c1af2f06..d1cfe9e445 100644
--- a/csharp/test/Apache.Arrow.Flight.TestWeb/Startup.cs
+++ b/csharp/test/Apache.Arrow.Flight.TestWeb/Startup.cs
@@ -13,15 +13,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Console;
namespace Apache.Arrow.Flight.TestWeb
{
@@ -35,6 +33,11 @@ namespace Apache.Arrow.Flight.TestWeb
.AddFlightServer<TestFlightServer>();
services.AddSingleton(new FlightStore());
+
+ // The integration tests rely on the port being written to the first line of stdout,
+ // so send all logging to stderr.
+ services.Configure<ConsoleLoggerOptions>(
+ o => o.LogToStandardErrorThreshold = LogLevel.Debug);
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
diff --git a/csharp/test/Apache.Arrow.Flight.TestWeb/TestFlightServer.cs b/csharp/test/Apache.Arrow.Flight.TestWeb/TestFlightServer.cs
index 4a72b73274..46c5460912 100644
--- a/csharp/test/Apache.Arrow.Flight.TestWeb/TestFlightServer.cs
+++ b/csharp/test/Apache.Arrow.Flight.TestWeb/TestFlightServer.cs
@@ -67,14 +67,16 @@ namespace Apache.Arrow.Flight.TestWeb
if(!_flightStore.Flights.TryGetValue(flightDescriptor, out var flightHolder))
{
- flightHolder = new FlightHolder(flightDescriptor, await requestStream.Schema, $"http://{context.Host}");
+ flightHolder = new FlightHolder(flightDescriptor, await requestStream.Schema, $"grpc+tcp://{context.Host}");
_flightStore.Flights.Add(flightDescriptor, flightHolder);
}
while (await requestStream.MoveNext())
{
- flightHolder.AddBatch(new RecordBatchWithMetadata(requestStream.Current, requestStream.ApplicationMetadata.FirstOrDefault()));
- await responseStream.WriteAsync(FlightPutResult.Empty);
+ var applicationMetadata = requestStream.ApplicationMetadata.FirstOrDefault();
+ flightHolder.AddBatch(new RecordBatchWithMetadata(requestStream.Current, applicationMetadata));
+ await responseStream.WriteAsync(
+ applicationMetadata == null ? FlightPutResult.Empty : new FlightPutResult(applicationMetadata));
}
}
diff --git a/dev/archery/archery/integration/datagen.py b/dev/archery/archery/integration/datagen.py
index 9f86d172dd..bc86296340 100644
--- a/dev/archery/archery/integration/datagen.py
+++ b/dev/archery/archery/integration/datagen.py
@@ -25,7 +25,7 @@ import tempfile
import numpy as np
from .util import frombytes, tobytes, random_bytes, random_utf8
-from .util import SKIP_C_SCHEMA, SKIP_C_ARRAY
+from .util import SKIP_C_SCHEMA, SKIP_C_ARRAY, SKIP_FLIGHT
def metadata_key_values(pairs):
@@ -1890,7 +1890,10 @@ def get_generated_json_files(tempdir=None):
return
file_objs = [
- generate_primitive_case([], name='primitive_no_batches'),
+ generate_primitive_case([], name='primitive_no_batches')
+ # TODO(https://github.com/apache/arrow/issues/44363)
+ .skip_format(SKIP_FLIGHT, 'C#'),
+
generate_primitive_case([17, 20], name='primitive'),
generate_primitive_case([0, 0, 0], name='primitive_zerolength'),
@@ -1952,16 +1955,22 @@ def get_generated_json_files(tempdir=None):
generate_dictionary_case()
# TODO(https://github.com/apache/arrow-nanoarrow/issues/622)
- .skip_tester('nanoarrow'),
+ .skip_tester('nanoarrow')
+ # TODO(https://github.com/apache/arrow/issues/38045)
+ .skip_format(SKIP_FLIGHT, 'C#'),
generate_dictionary_unsigned_case()
.skip_tester('nanoarrow')
- .skip_tester('Java'), # TODO(ARROW-9377)
+ .skip_tester('Java') # TODO(ARROW-9377)
+ # TODO(https://github.com/apache/arrow/issues/38045)
+ .skip_format(SKIP_FLIGHT, 'C#'),
generate_nested_dictionary_case()
# TODO(https://github.com/apache/arrow-nanoarrow/issues/622)
.skip_tester('nanoarrow')
- .skip_tester('Java'), # TODO(ARROW-7779)
+ .skip_tester('Java') # TODO(ARROW-7779)
+ # TODO(https://github.com/apache/arrow/issues/38045)
+ .skip_format(SKIP_FLIGHT, 'C#'),
generate_run_end_encoded_case()
.skip_tester('C#')
@@ -1988,7 +1997,9 @@ def get_generated_json_files(tempdir=None):
.skip_tester('nanoarrow')
# TODO: ensure the extension is registered in the C++ entrypoint
.skip_format(SKIP_C_SCHEMA, 'C++')
- .skip_format(SKIP_C_ARRAY, 'C++'),
+ .skip_format(SKIP_C_ARRAY, 'C++')
+ # TODO(https://github.com/apache/arrow/issues/38045)
+ .skip_format(SKIP_FLIGHT, 'C#'),
]
generated_paths = []
diff --git a/dev/archery/archery/integration/runner.py b/dev/archery/archery/integration/runner.py
index e276738846..378b17d75f 100644
--- a/dev/archery/archery/integration/runner.py
+++ b/dev/archery/archery/integration/runner.py
@@ -631,10 +631,13 @@ def run_all_tests(with_cpp=True, with_java=True, with_js=True,
flight_scenarios = [
Scenario(
"auth:basic_proto",
- description="Authenticate using the BasicAuth protobuf."),
+ description="Authenticate using the BasicAuth protobuf.",
+ skip_testers={"C#"},
+ ),
Scenario(
"middleware",
description="Ensure headers are propagated via middleware.",
+ skip_testers={"C#"},
),
Scenario(
"ordered",
@@ -689,12 +692,12 @@ def run_all_tests(with_cpp=True, with_java=True, with_js=True,
Scenario(
"flight_sql",
description="Ensure Flight SQL protocol is working as expected.",
- skip_testers={"Rust"}
+ skip_testers={"Rust", "C#"}
),
Scenario(
"flight_sql:extension",
description="Ensure Flight SQL extensions work as expected.",
- skip_testers={"Rust"}
+ skip_testers={"Rust", "C#"}
),
Scenario(
"flight_sql:ingestion",
diff --git a/dev/archery/archery/integration/tester_csharp.py b/dev/archery/archery/integration/tester_csharp.py
index 02ced0701d..50b3499fbf 100644
--- a/dev/archery/archery/integration/tester_csharp.py
+++ b/dev/archery/archery/integration/tester_csharp.py
@@ -17,6 +17,7 @@
from contextlib import contextmanager
import os
+import subprocess
from . import cdata
from .tester import Tester, CDataExporter, CDataImporter
@@ -25,12 +26,20 @@ from ..utils.source import ARROW_ROOT_DEFAULT
_ARTIFACTS_PATH = os.path.join(ARROW_ROOT_DEFAULT, "csharp/artifacts")
+_BUILD_SUBDIR = "Debug/net8.0"
_EXE_PATH = os.path.join(_ARTIFACTS_PATH,
"Apache.Arrow.IntegrationTest",
- "Debug/net8.0/Apache.Arrow.IntegrationTest",
+ _BUILD_SUBDIR,
+ "Apache.Arrow.IntegrationTest",
)
+_FLIGHT_EXE_PATH = os.path.join(_ARTIFACTS_PATH,
+ "Apache.Arrow.Flight.IntegrationTest",
+ _BUILD_SUBDIR,
+ "Apache.Arrow.Flight.IntegrationTest",
+ )
+
_clr_loaded = False
@@ -44,10 +53,10 @@ def _load_clr():
import clr
clr.AddReference(
f"{_ARTIFACTS_PATH}/Apache.Arrow.IntegrationTest/"
- f"Debug/net8.0/Apache.Arrow.IntegrationTest.dll")
+ f"{_BUILD_SUBDIR}/Apache.Arrow.IntegrationTest.dll")
clr.AddReference(
f"{_ARTIFACTS_PATH}/Apache.Arrow.Tests/"
- f"Debug/net8.0/Apache.Arrow.Tests.dll")
+ f"{_BUILD_SUBDIR}/Apache.Arrow.Tests.dll")
from Apache.Arrow.IntegrationTest import CDataInterface
CDataInterface.Initialize()
@@ -146,6 +155,8 @@ class CSharpCDataImporter(CDataImporter, _CDataBase):
class CSharpTester(Tester):
PRODUCER = True
CONSUMER = True
+ FLIGHT_SERVER = True
+ FLIGHT_CLIENT = True
C_DATA_SCHEMA_EXPORTER = True
C_DATA_SCHEMA_IMPORTER = True
C_DATA_ARRAY_EXPORTER = True
@@ -192,3 +203,43 @@ class CSharpTester(Tester):
def make_c_data_importer(self):
return CSharpCDataImporter(self.debug, self.args)
+
+ def flight_request(self, port, json_path=None, scenario_name=None):
+ cmd = [_FLIGHT_EXE_PATH, 'client', '--port', f'{port}']
+ if json_path:
+ cmd.extend(['--path', json_path])
+ elif scenario_name:
+ cmd.extend(['--scenario', scenario_name])
+ else:
+ raise TypeError("Must provide one of json_path or scenario_name")
+
+ if self.debug:
+ log(' '.join(cmd))
+ run_cmd(cmd)
+
+ @contextmanager
+ def flight_server(self, scenario_name=None):
+ cmd = [_FLIGHT_EXE_PATH, 'server']
+ if scenario_name:
+ cmd.extend(['--scenario', scenario_name])
+ if self.debug:
+ log(' '.join(cmd))
+ server = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ try:
+ output = server.stdout.readline().decode()
+ if not output.startswith("Server listening on "):
+ server.kill()
+ out, err = server.communicate()
+ raise RuntimeError(
+ '.NET Flight server did not start properly, '
+ 'stdout: \n{}\n\nstderr:\n{}\n'.format(
+ output + out.decode(), err.decode()
+ )
+ )
+ port = int(output.split(':')[-1])
+ yield port
+ finally:
+ server.kill()
+ server.wait(5)
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index e149c17981..dda1d36dc1 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -116,6 +116,7 @@ csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj
csharp/src/Apache.Arrow.Compression/Apache.Arrow.Compression.csproj
csharp/src/Apache.Arrow.Flight.Sql/Apache.Arrow.Flight.Sql.csproj
csharp/test/Apache.Arrow.Benchmarks/Apache.Arrow.Benchmarks.csproj
+csharp/test/Apache.Arrow.Flight.IntegrationTest/Apache.Arrow.Flight.IntegrationTest.csproj
csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj
csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj
csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj