You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by at...@apache.org on 2026/06/22 22:16:28 UTC
(iceberg-go) branch main updated: Custom namespace separator (#1253)
This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 94483396 Custom namespace separator (#1253)
94483396 is described below
commit 9448339625472e0b02928c7d6c92cb904741a184
Author: Alex Stephen <13...@users.noreply.github.com>
AuthorDate: Mon Jun 22 15:16:23 2026 -0700
Custom namespace separator (#1253)
We currently don't allow iceberg-go users to change the custom namespace
separator for multi-part namespaces.
I ran into this as an issue in the Terraform provider, where it appears
that the namespace separator for the REST Fixture has changed recently.
---
catalog/rest/rest.go | 88 +++++++++++++++++++++++++++-----------
catalog/rest/rest_internal_test.go | 83 +++++++++++++++++++++++++++++++++++
2 files changed, 146 insertions(+), 25 deletions(-)
diff --git a/catalog/rest/rest.go b/catalog/rest/rest.go
index 61b8b249..9146455b 100644
--- a/catalog/rest/rest.go
+++ b/catalog/rest/rest.go
@@ -70,9 +70,11 @@ const (
authorizationHeader = "Authorization"
bearerPrefix = "Bearer"
- namespaceSeparator = "\x1F"
keyPrefix = "prefix"
+ keyNamespaceSeparator = "namespace-separator"
+ defaultNamespaceSeparator = "%1F"
+
icebergRestSpecVersion = "0.14.1"
keyRestSigV4 = "rest.sigv4-enabled"
@@ -532,6 +534,8 @@ type Catalog struct {
name string
props iceberg.Properties
endpoints endpointSet
+
+ namespaceSeparator string
}
func newCatalogFromProps(ctx context.Context, name string, uri string, p iceberg.Properties) (*Catalog, error) {
@@ -743,6 +747,8 @@ func (r *Catalog) fetchConfig(ctx context.Context, opts *options) (*http.Client,
maps.Copy(cfg, toProps(opts))
maps.Copy(cfg, rsp.Overrides)
+ r.namespaceSeparator = cfg.Get(keyNamespaceSeparator, defaultNamespaceSeparator)
+
// Negotiate capabilities from the endpoints the server advertises, falling
// back to a backward-compatible default set when none are provided.
r.endpoints = resolveEndpoints(rsp.Endpoints, cfg.GetBool(keyViewEndpointsSupported, false))
@@ -803,7 +809,7 @@ func (r *Catalog) fetchTableCreds(ctx context.Context, ident []string, location
return nil, err
}
- ns, tbl, err := splitIdentForPath(ident)
+ ns, tbl, err := r.splitIdentForPath(ident)
if err != nil {
return nil, err
}
@@ -857,7 +863,7 @@ func (r *Catalog) listTablesPage(ctx context.Context, namespace table.Identifier
if !r.endpoints.allowed(endpointListTables) {
return nil, "", nil
}
- ns := strings.Join(namespace, namespaceSeparator)
+ ns := r.encodeNamespace(namespace)
path, err := endpointListTables.reqPath(ns)
if err != nil {
return nil, "", err
@@ -889,13 +895,45 @@ func (r *Catalog) listTablesPage(ctx context.Context, namespace table.Identifier
return out, rsp.NextPageToken, nil
}
-func splitIdentForPath(ident table.Identifier) (string, string, error) {
+func (r *Catalog) nsSeparator() string {
+ if r.namespaceSeparator == "" {
+ return defaultNamespaceSeparator
+ }
+
+ return r.namespaceSeparator
+}
+
+// encodeNamespace URL-encodes each namespace level and joins them with the
+// server-advertised, URL-encoded namespace separator for use as a REST path
+// segment. Mirrors RESTUtil.encodeNamespace in the Java implementation.
+func (r *Catalog) encodeNamespace(namespace table.Identifier) string {
+ encoded := make([]string, len(namespace))
+ for i, level := range namespace {
+ encoded[i] = url.PathEscape(level)
+ }
+
+ return strings.Join(encoded, r.nsSeparator())
+}
+
+// namespaceToQueryParam joins the raw namespace levels with the decoded
+// separator for use as a query-parameter value, which the HTTP layer then
+// percent-encodes. Mirrors RESTUtil.namespaceToQueryParam in Java.
+func (r *Catalog) namespaceToQueryParam(namespace table.Identifier) string {
+ sep, err := url.PathUnescape(r.nsSeparator())
+ if err != nil {
+ sep = r.nsSeparator()
+ }
+
+ return strings.Join(namespace, sep)
+}
+
+func (r *Catalog) splitIdentForPath(ident table.Identifier) (string, string, error) {
if len(ident) < 1 {
return "", "", fmt.Errorf("%w: missing namespace or invalid identifier %v",
catalog.ErrNoSuchTable, strings.Join(ident, "."))
}
- return strings.Join(catalog.NamespaceFromIdent(ident), namespaceSeparator), catalog.TableNameFromIdent(ident), nil
+ return r.encodeNamespace(catalog.NamespaceFromIdent(ident)), catalog.TableNameFromIdent(ident), nil
}
func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier, schema *iceberg.Schema, opts ...catalog.CreateTableOpt) (*table.Table, error) {
@@ -903,7 +941,7 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
return nil, err
}
- ns, tbl, err := splitIdentForPath(identifier)
+ ns, tbl, err := r.splitIdentForPath(identifier)
if err != nil {
return nil, err
}
@@ -1014,7 +1052,7 @@ func (r *Catalog) CommitTable(ctx context.Context, ident table.Identifier, requi
return nil, "", err
}
- ns, tblName, err := splitIdentForPath(ident)
+ ns, tblName, err := r.splitIdentForPath(ident)
if err != nil {
return nil, "", err
}
@@ -1129,7 +1167,7 @@ func (r *Catalog) RegisterTable(ctx context.Context, identifier table.Identifier
return nil, err
}
- ns, tbl, err := splitIdentForPath(identifier)
+ ns, tbl, err := r.splitIdentForPath(identifier)
if err != nil {
return nil, err
}
@@ -1166,7 +1204,7 @@ func (r *Catalog) LoadTable(ctx context.Context, identifier table.Identifier) (*
return nil, err
}
- ns, tbl, err := splitIdentForPath(identifier)
+ ns, tbl, err := r.splitIdentForPath(identifier)
if err != nil {
return nil, err
}
@@ -1196,7 +1234,7 @@ func (r *Catalog) UpdateTable(ctx context.Context, ident table.Identifier, requi
return nil, err
}
- ns, tbl, err := splitIdentForPath(ident)
+ ns, tbl, err := r.splitIdentForPath(ident)
if err != nil {
return nil, err
}
@@ -1240,7 +1278,7 @@ func (r *Catalog) DropTable(ctx context.Context, identifier table.Identifier) er
return err
}
- ns, tbl, err := splitIdentForPath(identifier)
+ ns, tbl, err := r.splitIdentForPath(identifier)
if err != nil {
return err
}
@@ -1266,7 +1304,7 @@ func (r *Catalog) PurgeTable(ctx context.Context, identifier table.Identifier) e
return err
}
- ns, tbl, err := splitIdentForPath(identifier)
+ ns, tbl, err := r.splitIdentForPath(identifier)
if err != nil {
return err
}
@@ -1350,7 +1388,7 @@ func (r *Catalog) DropNamespace(ctx context.Context, namespace table.Identifier)
return err
}
- path, err := endpointDeleteNamespace.reqPath(strings.Join(namespace, namespaceSeparator))
+ path, err := endpointDeleteNamespace.reqPath(r.encodeNamespace(namespace))
if err != nil {
return err
}
@@ -1398,7 +1436,7 @@ func (r *Catalog) listNamespacesPage(ctx context.Context, parent table.Identifie
v := url.Values{}
if len(parent) != 0 {
- v.Set("parent", strings.Join(parent, namespaceSeparator))
+ v.Set("parent", r.namespaceToQueryParam(parent))
}
if pageSize > 0 {
v.Set("pageSize", strconv.Itoa(pageSize))
@@ -1437,7 +1475,7 @@ func (r *Catalog) LoadNamespaceProperties(ctx context.Context, namespace table.I
Props iceberg.Properties `json:"properties"`
}
- path, err := endpointLoadNamespace.reqPath(strings.Join(namespace, namespaceSeparator))
+ path, err := endpointLoadNamespace.reqPath(r.encodeNamespace(namespace))
if err != nil {
return nil, err
}
@@ -1467,7 +1505,7 @@ func (r *Catalog) UpdateNamespaceProperties(ctx context.Context, namespace table
Updates iceberg.Properties `json:"updates"`
}
- ns := strings.Join(namespace, namespaceSeparator)
+ ns := r.encodeNamespace(namespace)
path, err := endpointUpdateNamespace.reqPath(ns)
if err != nil {
@@ -1498,7 +1536,7 @@ func (r *Catalog) CheckNamespaceExists(ctx context.Context, namespace table.Iden
return true, nil
}
- path, err := endpointNamespaceExists.reqPath(strings.Join(namespace, namespaceSeparator))
+ path, err := endpointNamespaceExists.reqPath(r.encodeNamespace(namespace))
if err != nil {
return false, err
}
@@ -1517,7 +1555,7 @@ func (r *Catalog) CheckNamespaceExists(ctx context.Context, namespace table.Iden
}
func (r *Catalog) CheckTableExists(ctx context.Context, identifier table.Identifier) (bool, error) {
- ns, tbl, err := splitIdentForPath(identifier)
+ ns, tbl, err := r.splitIdentForPath(identifier)
if err != nil {
return false, err
}
@@ -1590,7 +1628,7 @@ func (r *Catalog) listViewsPage(ctx context.Context, namespace table.Identifier,
if !r.endpoints.allowed(endpointListViews) {
return nil, "", nil
}
- ns := strings.Join(namespace, namespaceSeparator)
+ ns := r.encodeNamespace(namespace)
path, err := endpointListViews.reqPath(ns)
if err != nil {
return nil, "", err
@@ -1641,7 +1679,7 @@ func (r *Catalog) DropView(ctx context.Context, identifier table.Identifier) err
return err
}
- ns, view, err := splitIdentForPath(identifier)
+ ns, view, err := r.splitIdentForPath(identifier)
if err != nil {
return err
}
@@ -1658,7 +1696,7 @@ func (r *Catalog) DropView(ctx context.Context, identifier table.Identifier) err
}
func (r *Catalog) CheckViewExists(ctx context.Context, identifier table.Identifier) (bool, error) {
- ns, view, err := splitIdentForPath(identifier)
+ ns, view, err := r.splitIdentForPath(identifier)
if err != nil {
return false, err
}
@@ -1728,7 +1766,7 @@ func (r *Catalog) CreateView(ctx context.Context, identifier table.Identifier, v
return nil, err
}
- ns, viewName, err := splitIdentForPath(identifier)
+ ns, viewName, err := r.splitIdentForPath(identifier)
if err != nil {
return nil, err
}
@@ -1789,7 +1827,7 @@ func (r *Catalog) UpdateView(ctx context.Context, ident table.Identifier, requir
return nil, err
}
- ns, viewName, err := splitIdentForPath(ident)
+ ns, viewName, err := r.splitIdentForPath(ident)
if err != nil {
return nil, err
}
@@ -1834,7 +1872,7 @@ func (r *Catalog) RegisterView(ctx context.Context, identifier table.Identifier,
return nil, err
}
- ns, v, err := splitIdentForPath(identifier)
+ ns, v, err := r.splitIdentForPath(identifier)
if err != nil {
return nil, err
}
@@ -1871,7 +1909,7 @@ func (r *Catalog) LoadView(ctx context.Context, identifier table.Identifier) (*v
return nil, err
}
- ns, v, err := splitIdentForPath(identifier)
+ ns, v, err := r.splitIdentForPath(identifier)
if err != nil {
return nil, err
}
diff --git a/catalog/rest/rest_internal_test.go b/catalog/rest/rest_internal_test.go
index cc89d36a..6330280e 100644
--- a/catalog/rest/rest_internal_test.go
+++ b/catalog/rest/rest_internal_test.go
@@ -982,3 +982,86 @@ func TestToPropsSigv4RegionFallback(t *testing.T) {
assert.False(t, ok)
})
}
+
+func TestEncodeNamespace(t *testing.T) {
+ tests := []struct {
+ name string
+ separator string
+ namespace []string
+ wantPath string
+ wantQueryPart string
+ }{
+ {"default empty separator", "", []string{"a", "b"}, "a%1Fb", "a\x1fb"},
+ {"explicit unit separator", "%1F", []string{"a", "b"}, "a%1Fb", "a\x1fb"},
+ {"dot separator", "%2E", []string{"analytics", "prod"}, "analytics%2Eprod", "analytics.prod"},
+ {"single level", "%2E", []string{"db1"}, "db1", "db1"},
+ {"level needing escaping", "%2E", []string{"a/b", "c d"}, "a%2Fb%2Ec%20d", "a/b.c d"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ r := &Catalog{namespaceSeparator: tt.separator}
+ assert.Equal(t, tt.wantPath, r.encodeNamespace(tt.namespace))
+ assert.Equal(t, tt.wantQueryPart, r.namespaceToQueryParam(tt.namespace))
+ })
+ }
+}
+
+func TestNamespaceSeparatorFromConfig(t *testing.T) {
+ mux := http.NewServeMux()
+ srv := httptest.NewServer(mux)
+ defer srv.Close()
+
+ mux.HandleFunc("/v1/config", func(w http.ResponseWriter, r *http.Request) {
+ json.NewEncoder(w).Encode(map[string]any{
+ "defaults": map[string]any{},
+ "overrides": map[string]any{"namespace-separator": "%2E"},
+ })
+ })
+
+ var gotPath string
+ mux.HandleFunc("/v1/namespaces/", func(w http.ResponseWriter, r *http.Request) {
+ gotPath = r.URL.EscapedPath()
+ json.NewEncoder(w).Encode(map[string]any{
+ "namespace": []string{"analytics", "prod"},
+ "properties": map[string]any{"owner": "data-team"},
+ })
+ })
+
+ cat, err := NewCatalog(context.Background(), "rest", srv.URL)
+ require.NoError(t, err)
+ assert.Equal(t, "%2E", cat.namespaceSeparator)
+
+ props, err := cat.LoadNamespaceProperties(context.Background(), []string{"analytics", "prod"})
+ require.NoError(t, err)
+ assert.Equal(t, "/v1/namespaces/analytics%2Eprod", gotPath)
+ assert.Equal(t, "data-team", props["owner"])
+}
+
+func TestNamespaceSeparatorDefaultsToUnitSeparator(t *testing.T) {
+ mux := http.NewServeMux()
+ srv := httptest.NewServer(mux)
+ defer srv.Close()
+
+ mux.HandleFunc("/v1/config", func(w http.ResponseWriter, r *http.Request) {
+ json.NewEncoder(w).Encode(map[string]any{
+ "defaults": map[string]any{}, "overrides": map[string]any{},
+ })
+ })
+
+ var gotPath string
+ mux.HandleFunc("/v1/namespaces/", func(w http.ResponseWriter, r *http.Request) {
+ gotPath = r.URL.EscapedPath()
+ json.NewEncoder(w).Encode(map[string]any{
+ "namespace": []string{"a", "b"}, "properties": map[string]any{},
+ })
+ })
+
+ cat, err := NewCatalog(context.Background(), "rest", srv.URL)
+ require.NoError(t, err)
+ assert.Equal(t, "%1F", cat.namespaceSeparator)
+
+ _, err = cat.LoadNamespaceProperties(context.Background(), []string{"a", "b"})
+ require.NoError(t, err)
+ assert.Equal(t, "/v1/namespaces/a%1Fb", gotPath)
+}