You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/06/23 17:48:48 UTC
(iceberg-go) branch main updated: feat(catalog/hadoop): Allow opt-in for arbitrary schemes for Hadoop Catalog (#1268)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new d4d3e2e5 feat(catalog/hadoop): Allow opt-in for arbitrary schemes for Hadoop Catalog (#1268)
d4d3e2e5 is described below
commit d4d3e2e5c3e7ba372be14958c96e523ba8728448
Author: Colton Loftus <70...@users.noreply.github.com>
AuthorDate: Tue Jun 23 13:48:43 2026 -0400
feat(catalog/hadoop): Allow opt-in for arbitrary schemes for Hadoop Catalog (#1268)
Next step for #1096
This PR allows clients to opt in to use the hadoop catalog with
arbitrary schemes. However, they have to explicitly set
`allow-unsafe-commits`. I followed the guidance on this as specified
here:
https://github.com/apache/iceberg-go/issues/1096#issuecomment-4656700687
Note that this allows any arbitrary scheme. It is unclear if this should
be expand to just add support for blob stores?
Note, I was going to add an integration test for minio with hadoop but
as far as I can tell, I can't get the integration test runner to work
for me. Unclear if I am doing something wrong or if we maybe need to
make a change to this to use relative paths or something analogous? (I
see the Dockerfile is there)
Note that this is the current integration test file without any changes
to the runner.
```
> make integration-hadoop
go test -tags=integration -v -run="^TestHadoopIntegration" ./catalog/hadoop/...
=== RUN TestHadoopIntegration
2026/06/23 12:18:27 github.com/testcontainers/testcontainers-go - Connected to docker:
Server Version: 29.1.3
API Version: 1.52
Operating System: Docker Desktop
Total Memory: 7836 MB
Labels:
com.docker.desktop.address=unix:///Users/cloftus/Library/Containers/com.docker.docker/Data/docker-cli.sock
Testcontainers for Go Version: v0.42.0
Resolved Docker Host: unix:///var/run/docker.sock
Resolved Docker Socket Path: /var/run/docker.sock
Test SessionID: b94c401c54d8169c5101c563021f3b088636ea1765586a79ac332f17f6fff9fc
Test ProcessID: 7e1b2a5e-9f59-4ff3-a4bf-7b3afef90979
#1 [internal] load local bake definitions
#1 reading from stdin 642B done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 2B done
#2 DONE 0.0s
hadoop_integration_test.go:58:
Error Trace: /Users/cloftus/github/iceberg-go/catalog/hadoop/hadoop_integration_test.go:58
/Users/cloftus/go/pkg/mod/github.com/stretchr/testify@v1.11.1/suite/suite.go:211
/Users/cloftus/github/iceberg-go/catalog/hadoop/hadoop_integration_test.go:315
Error: Received unexpected error:
fail to up compose: compose up: failed to solve: failed to read dockerfile: open Dockerfile: no such file or directory
View build details: docker-desktop://dashboard/build/default/default/n0tmfi552z84ltchd6clbeftd
Test: TestHadoopIntegration
--- FAIL: TestHadoopIntegration (0.99s)
FAIL
FAIL github.com/apache/iceberg-go/catalog/hadoop 2.042s
FAIL
make: *** [integration-hadoop] Error 1
```
---
catalog/hadoop/hadoop.go | 14 +++++++++-----
catalog/hadoop/hadoop_test.go | 6 +++---
2 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/catalog/hadoop/hadoop.go b/catalog/hadoop/hadoop.go
index ff4af5cd..aab3b005 100644
--- a/catalog/hadoop/hadoop.go
+++ b/catalog/hadoop/hadoop.go
@@ -116,9 +116,10 @@ type Catalog struct {
}
// NewCatalog creates a new Hadoop catalog rooted at the given warehouse path.
-// Currently only local filesystem paths are supported. The warehouse directory
-// is not created on construction; it is created implicitly by the first
-// CreateNamespace call.
+// When using a local filesystem, the warehouse directory
+// is not created on construction; it is created implicitly by the first CreateNamespace
+// call. When using other schemes, the property `allow-unsafe-commits` must be set to
+// true since custom schemes and blob filesystems do not have the same atomicity guarantees.
func NewCatalog(name, warehouse string, props iceberg.Properties) (*Catalog, error) {
if warehouse == "" {
return nil, errors.New("hadoop catalog requires a warehouse path")
@@ -129,8 +130,11 @@ func NewCatalog(name, warehouse string, props iceberg.Properties) (*Catalog, err
return nil, fmt.Errorf("hadoop catalog: invalid warehouse path: %w", err)
}
- if u.Scheme != "" && u.Scheme != "file" {
- return nil, fmt.Errorf("hadoop catalog: unsupported warehouse scheme %q, must be file:// or a local path", u.Scheme)
+ isLocal := u.Scheme == "" || u.Scheme == "file"
+ allowUnsafeCommits := props.GetBool("allow-unsafe-commits", false)
+
+ if !isLocal && !allowUnsafeCommits {
+ return nil, fmt.Errorf("hadoop catalog: when using warehouse scheme %q, `allow-unsafe-commits` must be set to true", u.Scheme)
}
if u.Opaque != "" {
diff --git a/catalog/hadoop/hadoop_test.go b/catalog/hadoop/hadoop_test.go
index 06ece367..bfb64b51 100644
--- a/catalog/hadoop/hadoop_test.go
+++ b/catalog/hadoop/hadoop_test.go
@@ -91,14 +91,14 @@ func (s *HadoopCatalogTestSuite) TestNewCatalogStripsFilePrefix() {
s.Equal("/tmp/wh", cat.warehouse)
}
-func (s *HadoopCatalogTestSuite) TestNewCatalogRejectsNonFileScheme() {
+func (s *HadoopCatalogTestSuite) TestNewCatalogRequiresOptInForRemoteSchemes() {
_, err := NewCatalog("test", "s3://bucket/path", nil)
s.Require().Error(err)
- s.Contains(err.Error(), "unsupported warehouse scheme")
+ s.Contains(err.Error(), "`allow-unsafe-commits` must be set to true")
_, err = NewCatalog("test", "hdfs://namenode/warehouse", nil)
s.Require().Error(err)
- s.Contains(err.Error(), "unsupported warehouse scheme")
+ s.Contains(err.Error(), "`allow-unsafe-commits` must be set to true")
}
func (s *HadoopCatalogTestSuite) TestNamespaceToPathSingleLevel() {