You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by sn...@apache.org on 2026/05/30 07:26:50 UTC
(polaris) branch main updated: Donate Azurite and fake-gcs-server testcontainers (#4571)
This is an automated email from the ASF dual-hosted git repository.
snazy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 5ce04a2112 Donate Azurite and fake-gcs-server testcontainers (#4571)
5ce04a2112 is described below
commit 5ce04a21122e12fd8e331aa846653122b6ef2532
Author: Robert Stupp <sn...@snazy.de>
AuthorDate: Sat May 30 09:26:45 2026 +0200
Donate Azurite and fake-gcs-server testcontainers (#4571)
This PR donates reusable Testcontainers-based utilities for Azurite and fake-gcs-server to Polaris.
The new testcontainer modules provide local, containerized object-storage fixtures that can be used by Polaris tests without requiring external cloud services or credentials.
They are intended as an initial import/donation of the existing test infrastructure so Polaris can build follow-up integration work on top of a shared baseline.
This PR is intentionally scoped to the donation of these testcontainer utilities.
Cleanup, package renaming, additional documentation, broader test integration, or behavior changes should be handled in follow-up PRs.
---
LICENSE | 17 ++
bom/build.gradle.kts | 2 +
gradle/projects.main.properties | 2 +
tools/azurite-testcontainer/build.gradle.kts | 44 +++++
.../testing/azurite/ITAzuriteExtension.java | 78 ++++++++
.../azurite/ITAzuriteExtensionInstanceField.java | 67 +++++++
.../azurite/ITAzuriteExtensionParameters.java | 40 ++++
.../azurite/ITAzuriteExtensionStaticField.java | 51 +++++
.../org/projectnessie/testing/azurite/Azurite.java | 36 ++++
.../testing/azurite/AzuriteAccess.java | 50 +++++
.../testing/azurite/AzuriteContainer.java | 194 +++++++++++++++++++
.../testing/azurite/AzuriteExtension.java | 126 +++++++++++++
.../testing/azurite/Dockerfile-azurite-version | 22 +++
tools/gcs-testcontainer/build.gradle.kts | 43 +++++
.../projectnessie/testing/gcs/ITGcsExtension.java | 85 +++++++++
.../testing/gcs/ITGcsExtensionInstanceField.java | 69 +++++++
.../testing/gcs/ITGcsExtensionParameters.java | 46 +++++
.../testing/gcs/ITGcsExtensionStaticField.java | 51 +++++
.../java/org/projectnessie/testing/gcs/Gcs.java | 38 ++++
.../org/projectnessie/testing/gcs/GcsAccess.java | 42 +++++
.../projectnessie/testing/gcs/GcsContainer.java | 210 +++++++++++++++++++++
.../projectnessie/testing/gcs/GcsExtension.java | 124 ++++++++++++
.../testing/gcs/Dockerfile-fake-gcs-server-version | 22 +++
23 files changed, 1459 insertions(+)
diff --git a/LICENSE b/LICENSE
index 2981dbe521..0b53197e8a 100644
--- a/LICENSE
+++ b/LICENSE
@@ -325,6 +325,23 @@ This product includes code from Project Nessie.
* tools/config-docs/generator/src/test/java/tests/smallrye/SomeEnum.java
* tools/config-docs/generator/src/test/java/tests/smallrye/VeryNested.java
* tools/container-spec-helper/src/main/java/org/apache/polaris/containerspec/ContainerSpecHelper.java
+* tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionInstanceField.java
+* tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtension.java
+* tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionParameters.java
+* tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionStaticField.java
+* tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteAccess.java
+* tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteContainer.java
+* tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteExtension.java
+* tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/Azurite.java
+* tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionInstanceField.java
+* tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtension.java
+* tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionParameters.java
+* tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionStaticField.java
+* tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsAccess.java
+* tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsContainer.java
+* tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsExtension.java
+* tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/Gcs.java
+* tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/Minio.java
* tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/Minio.java
* tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioAccess.java
* tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioContainer.java
diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts
index 8a1fa16b90..ecc5f89233 100644
--- a/bom/build.gradle.kts
+++ b/bom/build.gradle.kts
@@ -30,6 +30,8 @@ dependencies {
api(project(":polaris-api-management-service"))
api(project(":polaris-container-spec-helper"))
+ api(project(":polaris-azurite-testcontainer"))
+ api(project(":polaris-gcs-testcontainer"))
api(project(":polaris-minio-testcontainer"))
api(project(":polaris-rustfs-testcontainer"))
api(project(":polaris-immutables"))
diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties
index 83327ffa08..322510c557 100644
--- a/gradle/projects.main.properties
+++ b/gradle/projects.main.properties
@@ -37,6 +37,8 @@ polaris-tests=integration-tests
aggregated-license-report=aggregated-license-report
polaris-immutables=tools/immutables
polaris-container-spec-helper=tools/container-spec-helper
+polaris-azurite-testcontainer=tools/azurite-testcontainer
+polaris-gcs-testcontainer=tools/gcs-testcontainer
polaris-minio-testcontainer=tools/minio-testcontainer
polaris-rustfs-testcontainer=tools/rustfs-testcontainer
polaris-hms-testcontainer=tools/hms-testcontainer
diff --git a/tools/azurite-testcontainer/build.gradle.kts b/tools/azurite-testcontainer/build.gradle.kts
new file mode 100644
index 0000000000..94685fb5dc
--- /dev/null
+++ b/tools/azurite-testcontainer/build.gradle.kts
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+ id("org.kordamp.gradle.jandex")
+ id("polaris-server")
+}
+
+dependencies {
+ api(platform(libs.testcontainers.bom))
+ api("org.testcontainers:testcontainers")
+
+ api(platform(libs.azuresdk.bom))
+ api("com.azure:azure-storage-file-datalake")
+ api("com.azure:azure-identity")
+
+ implementation(project(":polaris-container-spec-helper"))
+ implementation(libs.guava)
+
+ compileOnly(platform(libs.junit.bom))
+ compileOnly("org.junit.jupiter:junit-jupiter-api")
+}
+
+testing {
+ suites {
+ val intTest by registering(JvmTestSuite::class)
+ }
+}
diff --git a/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtension.java b/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtension.java
new file mode 100644
index 0000000000..ff773c860a
--- /dev/null
+++ b/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtension.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.azurite;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.azure.storage.file.datalake.DataLakeFileClient;
+import com.azure.storage.file.datalake.DataLakeServiceClient;
+import java.io.InputStream;
+import java.io.OutputStream;
+import org.assertj.core.api.SoftAssertions;
+import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
+import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith({AzuriteExtension.class, SoftAssertionsExtension.class})
+public class ITAzuriteExtension {
+ @InjectSoftAssertions private SoftAssertions soft;
+
+ @Test
+ public void smokeTest(
+ @Azurite(account = "myaccesskey", secret = "mysecretkey", storageContainer = "mybucket")
+ AzuriteAccess azurite)
+ throws Exception {
+ soft.assertThat(azurite.endpointHostPort()).isNotEmpty();
+ soft.assertThat(azurite.endpoint()).isNotEmpty().startsWith("http");
+ soft.assertThat(azurite.endpoint()).isNotEmpty().startsWith("http");
+
+ soft.assertThat(azurite.storageContainer()).isNotEmpty().isEqualTo("mybucket");
+
+ soft.assertThat(azurite.icebergProperties())
+ .containsEntry("io-impl", "org.apache.iceberg.azure.adlsv2.ADLSFileIO")
+ .containsEntry("adls.connection-string." + azurite.accountFq(), azurite.endpoint())
+ .containsEntry("adls.auth.shared-key.account.name", azurite.account())
+ .containsEntry("adls.auth.shared-key.account.key", azurite.secretBase64());
+
+ soft.assertThat(azurite.hadoopConfig())
+ .isNotNull()
+ .containsEntry("fs.azure.impl", "org.apache.hadoop.fs.azure.AzureNativeFileSystemStore")
+ .containsEntry("fs.AbstractFileSystem.azure.impl", "org.apache.hadoop.fs.azurebfs.Abfs")
+ .containsEntry("fs.azure.always.use.https", "false")
+ .containsEntry("fs.azure.abfs.endpoint", azurite.endpointHostPort())
+ .containsEntry("fs.azure.account.auth.type", "SharedKey")
+ .containsEntry("fs.azure.storage.emulator.account.name", azurite.account())
+ .containsEntry("fs.azure.account.key." + azurite.accountFq(), azurite.secretBase64());
+
+ DataLakeServiceClient client = azurite.serviceClient();
+ byte[] data = "hello world".getBytes(UTF_8);
+ String key = "some-key";
+
+ soft.assertThat(azurite.location("some-key"))
+ .isEqualTo("abfs://" + azurite.storageContainer() + "@" + azurite.accountFq() + "/" + key);
+
+ DataLakeFileClient fileClient =
+ client.getFileSystemClient(azurite.storageContainer()).getFileClient(key);
+ try (OutputStream output = fileClient.getOutputStream()) {
+ output.write(data);
+ }
+
+ try (InputStream input = fileClient.openInputStream().getInputStream()) {
+ soft.assertThat(input.readAllBytes()).isEqualTo(data);
+ }
+ }
+}
diff --git a/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionInstanceField.java b/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionInstanceField.java
new file mode 100644
index 0000000000..8a0c4c7153
--- /dev/null
+++ b/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionInstanceField.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.azurite;
+
+import org.assertj.core.api.SoftAssertions;
+import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
+import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.testcontainers.containers.GenericContainer;
+
+@ExtendWith({AzuriteExtension.class, SoftAssertionsExtension.class})
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+public class ITAzuriteExtensionInstanceField {
+ @InjectSoftAssertions private SoftAssertions soft;
+
+ @Azurite private AzuriteAccess azuriteInstanceField;
+
+ static AzuriteAccess memoizedInstanceInstance;
+
+ @Order(101)
+ @Test
+ public void fields1() {
+ soft.assertThat(azuriteInstanceField).isNotNull();
+
+ soft.assertThat(azuriteInstanceField.storageContainer()).isNotEmpty();
+
+ soft.assertThat(azuriteInstanceField.endpoint()).isNotEmpty();
+
+ soft.assertThat(azuriteInstanceField.endpointHostPort()).isNotEmpty();
+
+ memoizedInstanceInstance = azuriteInstanceField;
+ }
+
+ @Order(102)
+ @Test
+ public void fields2() {
+ soft.assertThat(azuriteInstanceField).isNotNull();
+
+ soft.assertThat(memoizedInstanceInstance).isNotNull().isNotSameAs(azuriteInstanceField);
+ soft.assertThat(((GenericContainer<?>) memoizedInstanceInstance).isRunning()).isFalse();
+
+ soft.assertThat(azuriteInstanceField.storageContainer()).isNotEmpty();
+
+ soft.assertThat(azuriteInstanceField).isNotSameAs(memoizedInstanceInstance);
+
+ soft.assertThat(azuriteInstanceField.storageContainer())
+ .isNotEmpty()
+ .isNotEqualTo(memoizedInstanceInstance.storageContainer());
+ }
+}
diff --git a/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionParameters.java b/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionParameters.java
new file mode 100644
index 0000000000..03262a5b11
--- /dev/null
+++ b/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionParameters.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.azurite;
+
+import org.assertj.core.api.SoftAssertions;
+import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
+import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith({AzuriteExtension.class, SoftAssertionsExtension.class})
+public class ITAzuriteExtensionParameters {
+ @InjectSoftAssertions private SoftAssertions soft;
+
+ @Test
+ public void fields2(@Azurite AzuriteAccess param1, @Azurite AzuriteAccess param2) {
+ soft.assertThat(param1).isNotNull();
+ soft.assertThat(param2).isNotNull();
+ soft.assertThat(param1).isNotSameAs(param2);
+
+ soft.assertThat(param1.storageContainer()).isNotEmpty();
+ soft.assertThat(param2.storageContainer()).isNotEmpty();
+
+ soft.assertThat(param1.storageContainer()).isNotEqualTo(param2.storageContainer());
+ soft.assertThat(param1.endpoint()).isNotEqualTo(param2.endpoint());
+ }
+}
diff --git a/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionStaticField.java b/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionStaticField.java
new file mode 100644
index 0000000000..1ef1ae1338
--- /dev/null
+++ b/tools/azurite-testcontainer/src/intTest/java/org/projectnessie/testing/azurite/ITAzuriteExtensionStaticField.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.azurite;
+
+import org.assertj.core.api.SoftAssertions;
+import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
+import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith({AzuriteExtension.class, SoftAssertionsExtension.class})
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+public class ITAzuriteExtensionStaticField {
+ @InjectSoftAssertions private SoftAssertions soft;
+
+ @Azurite private static AzuriteAccess azuriteStaticField;
+
+ static AzuriteAccess memoizedStaticInstance;
+
+ @Order(101)
+ @Test
+ public void fields1() {
+ soft.assertThat(azuriteStaticField).isNotNull();
+
+ soft.assertThat(azuriteStaticField.endpoint()).isNotEmpty();
+
+ memoizedStaticInstance = azuriteStaticField;
+ }
+
+ @Order(102)
+ @Test
+ public void fields2() {
+ soft.assertThat(azuriteStaticField).isSameAs(memoizedStaticInstance);
+ }
+}
diff --git a/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/Azurite.java b/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/Azurite.java
new file mode 100644
index 0000000000..efc839ff09
--- /dev/null
+++ b/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/Azurite.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.azurite;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target({ElementType.FIELD, ElementType.PARAMETER})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface Azurite {
+ /** Optional, use this file-system/storage-container name instead of a random one. */
+ String storageContainer() default DEFAULT;
+
+ String account() default DEFAULT;
+
+ String secret() default DEFAULT;
+
+ String DEFAULT = "azurite_default_value__";
+}
diff --git a/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteAccess.java b/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteAccess.java
new file mode 100644
index 0000000000..dc769b67d3
--- /dev/null
+++ b/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteAccess.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.azurite;
+
+import com.azure.storage.common.StorageSharedKeyCredential;
+import com.azure.storage.file.datalake.DataLakeServiceClient;
+import java.util.Map;
+
+public interface AzuriteAccess {
+ DataLakeServiceClient serviceClient();
+
+ String storageContainer();
+
+ String location(String path);
+
+ String endpoint();
+
+ String endpointHostPort();
+
+ String account();
+
+ String accountFq();
+
+ String secret();
+
+ String secretBase64();
+
+ StorageSharedKeyCredential credential();
+
+ Map<String, String> icebergProperties();
+
+ Map<String, String> hadoopConfig();
+
+ void createStorageContainer();
+
+ void deleteStorageContainer();
+}
diff --git a/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteContainer.java b/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteContainer.java
new file mode 100644
index 0000000000..cda4089287
--- /dev/null
+++ b/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteContainer.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.azurite;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.azure.storage.common.StorageSharedKeyCredential;
+import com.azure.storage.file.datalake.DataLakeServiceClient;
+import com.azure.storage.file.datalake.DataLakeServiceClientBuilder;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.polaris.containerspec.ContainerSpecHelper;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
+import org.testcontainers.utility.Base58;
+
+public class AzuriteContainer extends GenericContainer<AzuriteContainer>
+ implements AzuriteAccess, AutoCloseable {
+
+ private static final int DEFAULT_PORT = 10000; // default blob service port
+ private static final String LOG_WAIT_REGEX =
+ "Azurite Blob service is successfully listening at .*";
+
+ private final String storageContainer;
+ private final String account;
+ private final String accountFq;
+ private final String secret;
+ private final String secretBase64;
+
+ public AzuriteContainer() {
+ this(null, null, null, null);
+ }
+
+ public AzuriteContainer(String image, String storageContainer, String account, String secret) {
+ super(
+ ContainerSpecHelper.containerSpecHelper("azurite", AzuriteContainer.class)
+ .dockerImageName(image));
+ if (storageContainer == null) {
+ storageContainer = randomString("filesystem");
+ }
+ if (account == null) {
+ account = randomString("account");
+ }
+ if (secret == null) {
+ secret = randomString("secret");
+ }
+ this.storageContainer = storageContainer;
+ this.account = account;
+ this.accountFq = account + ".dfs.core.windows.net";
+ this.secret = secret;
+ this.secretBase64 = new String(Base64.getEncoder().encode(secret.getBytes(UTF_8)), UTF_8);
+
+ this.addExposedPort(DEFAULT_PORT);
+ this.setWaitStrategy(new LogMessageWaitStrategy().withRegEx(LOG_WAIT_REGEX));
+ this.addEnv("AZURITE_ACCOUNTS", account + ":" + this.secretBase64);
+ // --skipApiVersionCheck is necessary until Azurite supports the latest API version used by
+ // Azure SDK 1.3.4.
+ // See https://github.com/Azure/Azurite/issues/2623
+ this.setCommand(
+ "azurite",
+ "--blobHost",
+ "0.0.0.0",
+ "--tableHost",
+ "0.0.0.0",
+ "--queueHost",
+ "0.0.0.0",
+ "--skipApiVersionCheck");
+ }
+
+ @Override
+ public void start() {
+ super.start();
+
+ createStorageContainer();
+ }
+
+ @Override
+ public void createStorageContainer() {
+ serviceClient().createFileSystem(storageContainer);
+ }
+
+ @Override
+ public void deleteStorageContainer() {
+ serviceClient().deleteFileSystem(storageContainer);
+ }
+
+ @Override
+ public DataLakeServiceClient serviceClient() {
+ return new DataLakeServiceClientBuilder()
+ .endpoint(endpoint())
+ .credential(credential())
+ .buildClient();
+ }
+
+ @Override
+ public String storageContainer() {
+ return storageContainer;
+ }
+
+ @Override
+ public String location(String path) {
+ if (path.startsWith("/")) {
+ path = path.substring(1);
+ }
+ return String.format("abfs://%s@%s/%s", storageContainer, accountFq, path);
+ }
+
+ @Override
+ public String endpoint() {
+ return String.format("http://%s/%s", endpointHostPort(), account);
+ }
+
+ @Override
+ public String endpointHostPort() {
+ return String.format("%s:%d", getHost(), getMappedPort(DEFAULT_PORT));
+ }
+
+ @Override
+ public StorageSharedKeyCredential credential() {
+ return new StorageSharedKeyCredential(account, secretBase64);
+ }
+
+ @Override
+ public String account() {
+ return account;
+ }
+
+ @Override
+ public String accountFq() {
+ return accountFq;
+ }
+
+ @Override
+ public String secret() {
+ return secret;
+ }
+
+ @Override
+ public String secretBase64() {
+ return secretBase64;
+ }
+
+ @Override
+ public Map<String, String> icebergProperties() {
+ Map<String, String> r = new HashMap<>();
+ r.put("io-impl", "org.apache.iceberg.azure.adlsv2.ADLSFileIO");
+ r.put("adls.connection-string." + accountFq, endpoint());
+ r.put("adls.auth.shared-key.account.name", account);
+ r.put("adls.auth.shared-key.account.key", secretBase64);
+ return r;
+ }
+
+ @Override
+ public Map<String, String> hadoopConfig() {
+ Map<String, String> r = new HashMap<>();
+
+ r.put("fs.azure.impl", "org.apache.hadoop.fs.azure.AzureNativeFileSystemStore");
+ r.put("fs.AbstractFileSystem.azure.impl", "org.apache.hadoop.fs.azurebfs.Abfs");
+
+ r.put("fs.azure.always.use.https", "false");
+ r.put("fs.azure.abfs.endpoint", endpointHostPort());
+
+ r.put("fs.azure.test.emulator", "true");
+ r.put("fs.azure.storage.emulator.account.name", account);
+ r.put("fs.azure.account.auth.type", "SharedKey");
+ r.put("fs.azure.account.key." + accountFq, secretBase64);
+
+ return r;
+ }
+
+ private static String randomString(String prefix) {
+ return prefix + "-" + Base58.randomString(6).toLowerCase(Locale.ROOT);
+ }
+
+ @Override
+ public void close() {
+ stop();
+ }
+}
diff --git a/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteExtension.java b/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteExtension.java
new file mode 100644
index 0000000000..d70d6f7094
--- /dev/null
+++ b/tools/azurite-testcontainer/src/main/java/org/projectnessie/testing/azurite/AzuriteExtension.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.azurite;
+
+import static java.lang.String.format;
+import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled;
+import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled;
+import static org.junit.platform.commons.util.AnnotationUtils.findAnnotatedFields;
+import static org.junit.platform.commons.util.ReflectionUtils.makeAccessible;
+
+import java.lang.reflect.Field;
+import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ConditionEvaluationResult;
+import org.junit.jupiter.api.extension.ExecutionCondition;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+import org.junit.platform.commons.util.AnnotationUtils;
+import org.junit.platform.commons.util.ExceptionUtils;
+import org.junit.platform.commons.util.ReflectionUtils;
+
+public class AzuriteExtension
+ implements BeforeAllCallback, BeforeEachCallback, ParameterResolver, ExecutionCondition {
+ private static final ExtensionContext.Namespace NAMESPACE =
+ ExtensionContext.Namespace.create(AzuriteExtension.class);
+
+ @Override
+ public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) {
+ if (OS.current() == OS.LINUX) {
+ return enabled("Running on Linux");
+ }
+ if (OS.current() == OS.MAC) {
+ return enabled("Running on macOS");
+ }
+ return disabled(format("Disabled on %s", OS.current().name()));
+ }
+
+ @Override
+ public void beforeAll(ExtensionContext context) {
+ Class<?> testClass = context.getRequiredTestClass();
+
+ findAnnotatedFields(testClass, Azurite.class, ReflectionUtils::isStatic)
+ .forEach(field -> injectField(context, field));
+ }
+
+ @Override
+ public void beforeEach(ExtensionContext context) {
+ Class<?> testClass = context.getRequiredTestClass();
+
+ findAnnotatedFields(testClass, Azurite.class, ReflectionUtils::isNotStatic)
+ .forEach(field -> injectField(context, field));
+ }
+
+ private void injectField(ExtensionContext context, Field field) {
+ try {
+ Azurite azurite =
+ AnnotationUtils.findAnnotation(field, Azurite.class)
+ .orElseThrow(IllegalStateException::new);
+
+ AzuriteAccess container =
+ context
+ .getStore(NAMESPACE)
+ .computeIfAbsent(
+ field.toString(), x -> createContainer(azurite), AzuriteAccess.class);
+
+ makeAccessible(field).set(context.getTestInstance().orElse(null), container);
+ } catch (Throwable t) {
+ ExceptionUtils.throwAsUncheckedException(t);
+ }
+ }
+
+ @Override
+ public boolean supportsParameter(
+ ParameterContext parameterContext, ExtensionContext extensionContext)
+ throws ParameterResolutionException {
+ if (parameterContext.findAnnotation(Azurite.class).isEmpty()) {
+ return false;
+ }
+ return parameterContext.getParameter().getType().isAssignableFrom(AzuriteAccess.class);
+ }
+
+ @Override
+ public Object resolveParameter(
+ ParameterContext parameterContext, ExtensionContext extensionContext)
+ throws ParameterResolutionException {
+ return extensionContext
+ .getStore(NAMESPACE)
+ .computeIfAbsent(
+ AzuriteExtension.class.getName() + '#' + parameterContext.getParameter().getName(),
+ k -> {
+ Azurite azurite = parameterContext.findAnnotation(Azurite.class).get();
+ return createContainer(azurite);
+ },
+ AzuriteAccess.class);
+ }
+
+ private AzuriteAccess createContainer(Azurite azurite) {
+ String bucket = nonDefault(azurite.storageContainer());
+ String account = nonDefault(azurite.storageContainer());
+ String secret = nonDefault(azurite.storageContainer());
+ AzuriteContainer container =
+ new AzuriteContainer(null, bucket, account, secret).withStartupAttempts(5);
+ container.start();
+ return container;
+ }
+
+ private static String nonDefault(String s) {
+ return s.equals(Azurite.DEFAULT) ? null : s;
+ }
+}
diff --git a/tools/azurite-testcontainer/src/main/resources/org/projectnessie/testing/azurite/Dockerfile-azurite-version b/tools/azurite-testcontainer/src/main/resources/org/projectnessie/testing/azurite/Dockerfile-azurite-version
new file mode 100644
index 0000000000..9bad38b501
--- /dev/null
+++ b/tools/azurite-testcontainer/src/main/resources/org/projectnessie/testing/azurite/Dockerfile-azurite-version
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Dockerfile to provide the image name and tag to a test.
+# Version is managed by Renovate - do not edit.
+FROM mcr.microsoft.com/azure-storage/azurite:3.35.0
diff --git a/tools/gcs-testcontainer/build.gradle.kts b/tools/gcs-testcontainer/build.gradle.kts
new file mode 100644
index 0000000000..0e1c952203
--- /dev/null
+++ b/tools/gcs-testcontainer/build.gradle.kts
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+ id("org.kordamp.gradle.jandex")
+ id("polaris-server")
+}
+
+dependencies {
+ api(platform(libs.testcontainers.bom))
+ api("org.testcontainers:testcontainers")
+
+ api(platform(libs.google.cloud.storage.bom))
+ api("com.google.cloud:google-cloud-storage")
+
+ implementation(project(":polaris-container-spec-helper"))
+ implementation(libs.guava)
+
+ compileOnly(platform(libs.junit.bom))
+ compileOnly("org.junit.jupiter:junit-jupiter-api")
+}
+
+testing {
+ suites {
+ val intTest by registering(JvmTestSuite::class)
+ }
+}
diff --git a/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtension.java b/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtension.java
new file mode 100644
index 0000000000..b18f88b8f6
--- /dev/null
+++ b/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtension.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.gcs;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.cloud.WriteChannel;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import org.assertj.core.api.SoftAssertions;
+import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
+import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith({GcsExtension.class, SoftAssertionsExtension.class})
+public class ITGcsExtension {
+ @InjectSoftAssertions private SoftAssertions soft;
+
+ @Test
+ public void smokeTest(
+ @Gcs(bucket = "mybucket", oauth2token = "myoauth", projectId = "myproject") GcsAccess gcs)
+ throws Exception {
+ soft.assertThat(gcs.localAddress()).isNotEmpty();
+ soft.assertThat(gcs.baseUri()).isNotEmpty().startsWith("http");
+ soft.assertThat(gcs.bucketUri()).extracting(URI::getScheme).isEqualTo("gs");
+
+ soft.assertThat(gcs.bucket()).isNotEmpty().isEqualTo("mybucket");
+ soft.assertThat(gcs.oauth2token()).isNotEmpty().isEqualTo("myoauth");
+ soft.assertThat(gcs.projectId()).isNotEmpty().isEqualTo("myproject");
+
+ soft.assertThat(gcs.icebergProperties())
+ .containsEntry("io-impl", "org.apache.iceberg.gcp.gcs.GCSFileIO")
+ .containsEntry("gcs.project-id", gcs.projectId())
+ .containsEntry("gcs.service.host", gcs.baseUri())
+ .containsEntry("gcs.oauth2.token", gcs.oauth2token());
+
+ soft.assertThat(gcs.hadoopConfig())
+ .isNotNull()
+ .containsEntry("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem")
+ .containsEntry(
+ "fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS")
+ .containsEntry("fs.gs.storage.root.url", gcs.baseUri())
+ .containsEntry("fs.gs.project.id", gcs.projectId());
+
+ try (Storage storage = gcs.newStorage()) {
+ BlobId blobId = BlobId.of(gcs.bucket(), "some-key");
+ byte[] data = "hello world".getBytes(UTF_8);
+
+ soft.assertThat(gcs.bucketUri(blobId.getName()))
+ .isEqualTo(URI.create("gs://" + gcs.bucket() + "/" + blobId.getName()));
+
+ storage.create(BlobInfo.newBuilder(blobId).setContentType("text/plain").build(), data);
+ soft.assertThat(storage.readAllBytes(blobId)).isEqualTo(data);
+ soft.assertThat(storage.get(blobId)).extracting(Blob::getContentType).isEqualTo("text/plain");
+
+ blobId = BlobId.of(gcs.bucket(), "other-key");
+ try (WriteChannel channel =
+ storage.writer(BlobInfo.newBuilder(blobId).setContentType("text/plain").build())) {
+ channel.write(ByteBuffer.wrap(data));
+ }
+ soft.assertThat(storage.readAllBytes(blobId)).isEqualTo(data);
+ // Looks like fake-gcs-server has a bug and does not respect the content-type when using
+ // `Storage.writer()`
+ // soft.assertThat(storage.get(blobId)).extracting(Blob::getContentType).isEqualTo("text/plain");
+ }
+ }
+}
diff --git a/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionInstanceField.java b/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionInstanceField.java
new file mode 100644
index 0000000000..05d9cf0b42
--- /dev/null
+++ b/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionInstanceField.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.gcs;
+
+import org.assertj.core.api.SoftAssertions;
+import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
+import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.condition.EnabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.testcontainers.containers.GenericContainer;
+
+@ExtendWith({GcsExtension.class, SoftAssertionsExtension.class})
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+@EnabledOnOs(
+ value = OS.LINUX,
+ disabledReason =
+ "127.x.x.x addresses needed for multiple container instances are only available on Linux")
+public class ITGcsExtensionInstanceField {
+ @InjectSoftAssertions private SoftAssertions soft;
+
+ @Gcs private GcsAccess gcsInstanceField;
+
+ static GcsAccess memoizedInstanceInstance;
+
+ @Order(101)
+ @Test
+ public void fields1() {
+ soft.assertThat(gcsInstanceField).isNotNull();
+
+ soft.assertThat(gcsInstanceField.bucket()).isNotEmpty();
+
+ memoizedInstanceInstance = gcsInstanceField;
+ }
+
+ @Order(102)
+ @Test
+ public void fields2() {
+ soft.assertThat(gcsInstanceField).isNotNull();
+
+ soft.assertThat(memoizedInstanceInstance).isNotNull().isNotSameAs(gcsInstanceField);
+ soft.assertThat(((GenericContainer<?>) memoizedInstanceInstance).isRunning()).isFalse();
+
+ soft.assertThat(gcsInstanceField.bucket()).isNotEmpty();
+
+ soft.assertThat(gcsInstanceField).isNotSameAs(memoizedInstanceInstance);
+
+ soft.assertThat(gcsInstanceField.bucket())
+ .isNotEmpty()
+ .isNotEqualTo(memoizedInstanceInstance.bucket());
+ }
+}
diff --git a/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionParameters.java b/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionParameters.java
new file mode 100644
index 0000000000..eee8da7f84
--- /dev/null
+++ b/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionParameters.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.gcs;
+
+import org.assertj.core.api.SoftAssertions;
+import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
+import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith({GcsExtension.class, SoftAssertionsExtension.class})
+@EnabledOnOs(
+ value = OS.LINUX,
+ disabledReason =
+ "127.x.x.x addresses needed for multiple container instances are only available on Linux")
+public class ITGcsExtensionParameters {
+ @InjectSoftAssertions private SoftAssertions soft;
+
+ @Test
+ public void fields2(@Gcs GcsAccess param1, @Gcs GcsAccess param2) {
+ soft.assertThat(param1).isNotNull();
+ soft.assertThat(param2).isNotNull();
+ soft.assertThat(param1).isNotSameAs(param2);
+
+ soft.assertThat(param1.bucket()).isNotEmpty();
+ soft.assertThat(param2.bucket()).isNotEmpty();
+
+ soft.assertThat(param1.bucket()).isNotEqualTo(param2.bucket());
+ soft.assertThat(param1.baseUri()).isNotEqualTo(param2.baseUri());
+ }
+}
diff --git a/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionStaticField.java b/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionStaticField.java
new file mode 100644
index 0000000000..368d51a08a
--- /dev/null
+++ b/tools/gcs-testcontainer/src/intTest/java/org/projectnessie/testing/gcs/ITGcsExtensionStaticField.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.gcs;
+
+import org.assertj.core.api.SoftAssertions;
+import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
+import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith({GcsExtension.class, SoftAssertionsExtension.class})
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+public class ITGcsExtensionStaticField {
+ @InjectSoftAssertions private SoftAssertions soft;
+
+ @Gcs private static GcsAccess gcsStaticField;
+
+ static GcsAccess memoizedStaticInstance;
+
+ @Order(101)
+ @Test
+ public void fields1() {
+ soft.assertThat(gcsStaticField).isNotNull();
+
+ soft.assertThat(gcsStaticField.bucket()).isNotEmpty();
+
+ memoizedStaticInstance = gcsStaticField;
+ }
+
+ @Order(102)
+ @Test
+ public void fields2() {
+ soft.assertThat(gcsStaticField).isSameAs(memoizedStaticInstance);
+ }
+}
diff --git a/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/Gcs.java b/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/Gcs.java
new file mode 100644
index 0000000000..734de51b9c
--- /dev/null
+++ b/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/Gcs.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.gcs;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target({ElementType.FIELD, ElementType.PARAMETER})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface Gcs {
+ /** Optional, use this project ID instead of a random one. */
+ String projectId() default DEFAULT;
+
+ /** Optional, use this OAuth 2 token instead of a random one. */
+ String oauth2token() default DEFAULT;
+
+ /** Optional, use this bucket instead of a random one. */
+ String bucket() default DEFAULT;
+
+ String DEFAULT = "gcs_default_value__";
+}
diff --git a/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsAccess.java b/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsAccess.java
new file mode 100644
index 0000000000..54ee7327a4
--- /dev/null
+++ b/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsAccess.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.gcs;
+
+import com.google.cloud.storage.Storage;
+import java.net.URI;
+import java.util.Map;
+
+public interface GcsAccess {
+ String baseUri();
+
+ String localAddress();
+
+ String oauth2token();
+
+ String bucket();
+
+ URI bucketUri();
+
+ String projectId();
+
+ Storage newStorage();
+
+ Map<String, String> hadoopConfig();
+
+ Map<String, String> icebergProperties();
+
+ URI bucketUri(String key);
+}
diff --git a/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsContainer.java b/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsContainer.java
new file mode 100644
index 0000000000..6a7929a6e6
--- /dev/null
+++ b/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsContainer.java
@@ -0,0 +1,210 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.gcs;
+
+import static java.lang.String.format;
+import static java.util.Collections.singletonList;
+
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.OAuth2Credentials;
+import com.google.cloud.storage.BucketInfo;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.polaris.containerspec.ContainerSpecHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
+import org.testcontainers.utility.Base58;
+
+/**
+ * Provides a test container for Google Cloud Storage, using the {@code
+ * docker.io/fsouza/fake-gcs-server} image.
+ *
+ * <p>This container must use a static TCP port, see below. To at least mitigate the chance that two
+ * concurrently running containers share the same IP:PORT combination, this implementation uses a
+ * random IP in the range 127.0.0.1 to 127.255.255.254. This does not work on Windows and Mac.
+ *
+ * <p>It would be really nice to test GCS using the Fake GCS server and dynamic ports, but that's
+ * impossible, because of the GCS upload protocol itself. GCS uploads are initiated using one HTTP
+ * request, which returns a {@code Location} header, which contains the URL for the upload that the
+ * client must use. Problem is that a Fake server running inside a Docker container cannot infer the
+ * dynamically mapped port for this test, because the initial HTTP request does not have a {@code
+ * Host} header.
+ */
+public class GcsContainer extends GenericContainer<GcsContainer>
+ implements GcsAccess, AutoCloseable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GcsContainer.class);
+ public static final int PORT = 4443;
+
+ private final String localAddress;
+ private final String oauth2token;
+ private final String bucket;
+ private final String projectId;
+
+ public GcsContainer() {
+ this(null, null, null, null);
+ }
+
+ @SuppressWarnings("resource")
+ public GcsContainer(String image, String bucket, String projectId, String oauth2token) {
+ super(
+ ContainerSpecHelper.containerSpecHelper("fake-gcs-server", GcsContainer.class)
+ .dockerImageName(image));
+
+ ThreadLocalRandom rand = ThreadLocalRandom.current();
+ boolean isMac = System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("mac");
+ boolean isWindows = System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("windows");
+ localAddress =
+ isMac || isWindows
+ ? "127.0.0.1"
+ : format("127.%d.%d.%d", rand.nextInt(256), rand.nextInt(256), rand.nextInt(1, 255));
+ if (oauth2token == null) {
+ oauth2token = randomString("token");
+ }
+ if (bucket == null) {
+ bucket = randomString("bucket");
+ }
+ if (projectId == null) {
+ projectId = randomString("project");
+ }
+ this.oauth2token = oauth2token;
+ this.bucket = bucket;
+ this.projectId = projectId;
+
+ withNetworkAliases(randomString("fake-gcs"));
+ withLogConsumer(c -> LOGGER.info("[FAKE-GCS] {}", c.getUtf8StringWithoutLineEnding()));
+ withCommand(
+ "-scheme",
+ "http",
+ "-public-host",
+ localAddress,
+ "-log-level",
+ "info",
+ "-external-url",
+ format("http://%s:%d/", localAddress, PORT));
+ setWaitStrategy(new LogMessageWaitStrategy().withRegEx(".*server started at .*"));
+
+ addExposedPort(PORT);
+
+ // STATIC PORT BINDING!
+ setPortBindings(singletonList(localAddress + ":" + PORT + ":" + PORT));
+ }
+
+ @Override
+ public void start() {
+ super.start();
+
+ try (Storage storage = newStorage()) {
+ storage.create(BucketInfo.newBuilder(bucket).build());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public String baseUri() {
+ return format("http://%s:%d", localAddress(), gcsPort());
+ }
+
+ @Override
+ public String localAddress() {
+ return localAddress;
+ }
+
+ @Override
+ public String oauth2token() {
+ return oauth2token;
+ }
+
+ @Override
+ public String bucket() {
+ return bucket;
+ }
+
+ @Override
+ public String projectId() {
+ return projectId;
+ }
+
+ @Override
+ public Storage newStorage() {
+ return StorageOptions.http()
+ .setHost(baseUri())
+ .setCredentials(OAuth2Credentials.create(new AccessToken(oauth2token, null)))
+ .setProjectId(projectId)
+ .build()
+ .getService();
+ }
+
+ @Override
+ public URI bucketUri() {
+ return bucketUri("");
+ }
+
+ @Override
+ public URI bucketUri(String key) {
+ if (key.startsWith("/")) {
+ key = key.substring(1);
+ }
+ return URI.create(format("gs://%s/%s", bucket, key));
+ }
+
+ private int gcsPort() {
+ return getMappedPort(PORT);
+ }
+
+ private static String randomString(String prefix) {
+ return prefix + "-" + Base58.randomString(6).toLowerCase(Locale.ROOT);
+ }
+
+ @Override
+ public Map<String, String> hadoopConfig() {
+ Map<String, String> r = new HashMap<>();
+ // See https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcs/CONFIGURATION.md
+ r.put("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem");
+ r.put("fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS");
+ r.put("fs.gs.storage.root.url", baseUri());
+ r.put("fs.gs.project.id", projectId);
+ // TODO the docs don't mention anything about using an OAuth2 token :(
+ r.put("fs.gs.auth.type", "USER_CREDENTIALS");
+ r.put("fs.gs.auth.client.id", "foo");
+ r.put("fs.gs.auth.client.secret", "bar");
+ r.put("fs.gs.auth.refresh.token", "baz");
+ return r;
+ }
+
+ @Override
+ public Map<String, String> icebergProperties() {
+ Map<String, String> r = new HashMap<>();
+ r.put("io-impl", "org.apache.iceberg.gcp.gcs.GCSFileIO");
+ r.put("gcs.project-id", projectId);
+ r.put("gcs.service.host", baseUri());
+ r.put("gcs.oauth2.token", oauth2token);
+ return r;
+ }
+
+ @Override
+ public void close() {
+ stop();
+ }
+}
diff --git a/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsExtension.java b/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsExtension.java
new file mode 100644
index 0000000000..fd8d369232
--- /dev/null
+++ b/tools/gcs-testcontainer/src/main/java/org/projectnessie/testing/gcs/GcsExtension.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.testing.gcs;
+
+import static java.lang.String.format;
+import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled;
+import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled;
+import static org.junit.platform.commons.util.AnnotationUtils.findAnnotatedFields;
+import static org.junit.platform.commons.util.ReflectionUtils.makeAccessible;
+
+import java.lang.reflect.Field;
+import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ConditionEvaluationResult;
+import org.junit.jupiter.api.extension.ExecutionCondition;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+import org.junit.platform.commons.util.AnnotationUtils;
+import org.junit.platform.commons.util.ExceptionUtils;
+import org.junit.platform.commons.util.ReflectionUtils;
+
+public class GcsExtension
+ implements BeforeAllCallback, BeforeEachCallback, ParameterResolver, ExecutionCondition {
+ private static final ExtensionContext.Namespace NAMESPACE =
+ ExtensionContext.Namespace.create(GcsExtension.class);
+
+ @Override
+ public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) {
+ if (OS.current() == OS.LINUX) {
+ return enabled("Running on Linux");
+ }
+ if (OS.current() == OS.MAC) {
+ return enabled("Running on macOS");
+ }
+ return disabled(format("Disabled on %s", OS.current().name()));
+ }
+
+ @Override
+ public void beforeAll(ExtensionContext context) {
+ Class<?> testClass = context.getRequiredTestClass();
+
+ findAnnotatedFields(testClass, Gcs.class, ReflectionUtils::isStatic)
+ .forEach(field -> injectField(context, field));
+ }
+
+ @Override
+ public void beforeEach(ExtensionContext context) {
+ Class<?> testClass = context.getRequiredTestClass();
+
+ findAnnotatedFields(testClass, Gcs.class, ReflectionUtils::isNotStatic)
+ .forEach(field -> injectField(context, field));
+ }
+
+ private void injectField(ExtensionContext context, Field field) {
+ try {
+ Gcs gcs =
+ AnnotationUtils.findAnnotation(field, Gcs.class).orElseThrow(IllegalStateException::new);
+
+ GcsAccess container =
+ context
+ .getStore(NAMESPACE)
+ .computeIfAbsent(field.toString(), x -> createContainer(gcs), GcsAccess.class);
+
+ makeAccessible(field).set(context.getTestInstance().orElse(null), container);
+ } catch (Throwable t) {
+ ExceptionUtils.throwAsUncheckedException(t);
+ }
+ }
+
+ @Override
+ public boolean supportsParameter(
+ ParameterContext parameterContext, ExtensionContext extensionContext)
+ throws ParameterResolutionException {
+ if (parameterContext.findAnnotation(Gcs.class).isEmpty()) {
+ return false;
+ }
+ return parameterContext.getParameter().getType().isAssignableFrom(GcsAccess.class);
+ }
+
+ @Override
+ public Object resolveParameter(
+ ParameterContext parameterContext, ExtensionContext extensionContext)
+ throws ParameterResolutionException {
+ return extensionContext
+ .getStore(NAMESPACE)
+ .computeIfAbsent(
+ GcsExtension.class.getName() + '#' + parameterContext.getParameter().getName(),
+ k -> {
+ Gcs gcs = parameterContext.findAnnotation(Gcs.class).get();
+ return createContainer(gcs);
+ },
+ GcsAccess.class);
+ }
+
+ private GcsAccess createContainer(Gcs gcs) {
+ String projectId = nonDefault(gcs.projectId());
+ String oauth2token = nonDefault(gcs.oauth2token());
+ String bucket = nonDefault(gcs.bucket());
+ GcsContainer container =
+ new GcsContainer(null, bucket, projectId, oauth2token).withStartupAttempts(5);
+ container.start();
+ return container;
+ }
+
+ private static String nonDefault(String s) {
+ return s.equals(Gcs.DEFAULT) ? null : s;
+ }
+}
diff --git a/tools/gcs-testcontainer/src/main/resources/org/projectnessie/testing/gcs/Dockerfile-fake-gcs-server-version b/tools/gcs-testcontainer/src/main/resources/org/projectnessie/testing/gcs/Dockerfile-fake-gcs-server-version
new file mode 100644
index 0000000000..bbde1441fb
--- /dev/null
+++ b/tools/gcs-testcontainer/src/main/resources/org/projectnessie/testing/gcs/Dockerfile-fake-gcs-server-version
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Dockerfile to provide the image name and tag to a test.
+# Version is managed by Renovate - do not edit.
+FROM docker.io/fsouza/fake-gcs-server:1.54.0