You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by at...@apache.org on 2026/06/22 22:16:28 UTC

(iceberg-go) branch main updated: Custom namespace separator (#1253)

This is an automated email from the ASF dual-hosted git repository.

laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 94483396 Custom namespace separator (#1253)
94483396 is described below

commit 9448339625472e0b02928c7d6c92cb904741a184
Author: Alex Stephen <13...@users.noreply.github.com>
AuthorDate: Mon Jun 22 15:16:23 2026 -0700

    Custom namespace separator (#1253)
    
    We currently don't allow iceberg-go users to change the custom namespace
    separator for multi-part namespaces.
    
    I ran into this as an issue in the Terraform provider, where it appears
    that the namespace separator for the REST Fixture has changed recently.
---
 catalog/rest/rest.go               | 88 +++++++++++++++++++++++++++-----------
 catalog/rest/rest_internal_test.go | 83 +++++++++++++++++++++++++++++++++++
 2 files changed, 146 insertions(+), 25 deletions(-)

diff --git a/catalog/rest/rest.go b/catalog/rest/rest.go
index 61b8b249..9146455b 100644
--- a/catalog/rest/rest.go
+++ b/catalog/rest/rest.go
@@ -70,9 +70,11 @@ const (
 
 	authorizationHeader = "Authorization"
 	bearerPrefix        = "Bearer"
-	namespaceSeparator  = "\x1F"
 	keyPrefix           = "prefix"
 
+	keyNamespaceSeparator     = "namespace-separator"
+	defaultNamespaceSeparator = "%1F"
+
 	icebergRestSpecVersion = "0.14.1"
 
 	keyRestSigV4        = "rest.sigv4-enabled"
@@ -532,6 +534,8 @@ type Catalog struct {
 	name      string
 	props     iceberg.Properties
 	endpoints endpointSet
+
+	namespaceSeparator string
 }
 
 func newCatalogFromProps(ctx context.Context, name string, uri string, p iceberg.Properties) (*Catalog, error) {
@@ -743,6 +747,8 @@ func (r *Catalog) fetchConfig(ctx context.Context, opts *options) (*http.Client,
 	maps.Copy(cfg, toProps(opts))
 	maps.Copy(cfg, rsp.Overrides)
 
+	r.namespaceSeparator = cfg.Get(keyNamespaceSeparator, defaultNamespaceSeparator)
+
 	// Negotiate capabilities from the endpoints the server advertises, falling
 	// back to a backward-compatible default set when none are provided.
 	r.endpoints = resolveEndpoints(rsp.Endpoints, cfg.GetBool(keyViewEndpointsSupported, false))
@@ -803,7 +809,7 @@ func (r *Catalog) fetchTableCreds(ctx context.Context, ident []string, location
 		return nil, err
 	}
 
-	ns, tbl, err := splitIdentForPath(ident)
+	ns, tbl, err := r.splitIdentForPath(ident)
 	if err != nil {
 		return nil, err
 	}
@@ -857,7 +863,7 @@ func (r *Catalog) listTablesPage(ctx context.Context, namespace table.Identifier
 	if !r.endpoints.allowed(endpointListTables) {
 		return nil, "", nil
 	}
-	ns := strings.Join(namespace, namespaceSeparator)
+	ns := r.encodeNamespace(namespace)
 	path, err := endpointListTables.reqPath(ns)
 	if err != nil {
 		return nil, "", err
@@ -889,13 +895,45 @@ func (r *Catalog) listTablesPage(ctx context.Context, namespace table.Identifier
 	return out, rsp.NextPageToken, nil
 }
 
-func splitIdentForPath(ident table.Identifier) (string, string, error) {
+func (r *Catalog) nsSeparator() string {
+	if r.namespaceSeparator == "" {
+		return defaultNamespaceSeparator
+	}
+
+	return r.namespaceSeparator
+}
+
+// encodeNamespace URL-encodes each namespace level and joins them with the
+// server-advertised, URL-encoded namespace separator for use as a REST path
+// segment. Mirrors RESTUtil.encodeNamespace in the Java implementation.
+func (r *Catalog) encodeNamespace(namespace table.Identifier) string {
+	encoded := make([]string, len(namespace))
+	for i, level := range namespace {
+		encoded[i] = url.PathEscape(level)
+	}
+
+	return strings.Join(encoded, r.nsSeparator())
+}
+
+// namespaceToQueryParam joins the raw namespace levels with the decoded
+// separator for use as a query-parameter value, which the HTTP layer then
+// percent-encodes. Mirrors RESTUtil.namespaceToQueryParam in Java.
+func (r *Catalog) namespaceToQueryParam(namespace table.Identifier) string {
+	sep, err := url.PathUnescape(r.nsSeparator())
+	if err != nil {
+		sep = r.nsSeparator()
+	}
+
+	return strings.Join(namespace, sep)
+}
+
+func (r *Catalog) splitIdentForPath(ident table.Identifier) (string, string, error) {
 	if len(ident) < 1 {
 		return "", "", fmt.Errorf("%w: missing namespace or invalid identifier %v",
 			catalog.ErrNoSuchTable, strings.Join(ident, "."))
 	}
 
-	return strings.Join(catalog.NamespaceFromIdent(ident), namespaceSeparator), catalog.TableNameFromIdent(ident), nil
+	return r.encodeNamespace(catalog.NamespaceFromIdent(ident)), catalog.TableNameFromIdent(ident), nil
 }
 
 func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier, schema *iceberg.Schema, opts ...catalog.CreateTableOpt) (*table.Table, error) {
@@ -903,7 +941,7 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		return nil, err
 	}
 
-	ns, tbl, err := splitIdentForPath(identifier)
+	ns, tbl, err := r.splitIdentForPath(identifier)
 	if err != nil {
 		return nil, err
 	}
@@ -1014,7 +1052,7 @@ func (r *Catalog) CommitTable(ctx context.Context, ident table.Identifier, requi
 		return nil, "", err
 	}
 
-	ns, tblName, err := splitIdentForPath(ident)
+	ns, tblName, err := r.splitIdentForPath(ident)
 	if err != nil {
 		return nil, "", err
 	}
@@ -1129,7 +1167,7 @@ func (r *Catalog) RegisterTable(ctx context.Context, identifier table.Identifier
 		return nil, err
 	}
 
-	ns, tbl, err := splitIdentForPath(identifier)
+	ns, tbl, err := r.splitIdentForPath(identifier)
 	if err != nil {
 		return nil, err
 	}
@@ -1166,7 +1204,7 @@ func (r *Catalog) LoadTable(ctx context.Context, identifier table.Identifier) (*
 		return nil, err
 	}
 
-	ns, tbl, err := splitIdentForPath(identifier)
+	ns, tbl, err := r.splitIdentForPath(identifier)
 	if err != nil {
 		return nil, err
 	}
@@ -1196,7 +1234,7 @@ func (r *Catalog) UpdateTable(ctx context.Context, ident table.Identifier, requi
 		return nil, err
 	}
 
-	ns, tbl, err := splitIdentForPath(ident)
+	ns, tbl, err := r.splitIdentForPath(ident)
 	if err != nil {
 		return nil, err
 	}
@@ -1240,7 +1278,7 @@ func (r *Catalog) DropTable(ctx context.Context, identifier table.Identifier) er
 		return err
 	}
 
-	ns, tbl, err := splitIdentForPath(identifier)
+	ns, tbl, err := r.splitIdentForPath(identifier)
 	if err != nil {
 		return err
 	}
@@ -1266,7 +1304,7 @@ func (r *Catalog) PurgeTable(ctx context.Context, identifier table.Identifier) e
 		return err
 	}
 
-	ns, tbl, err := splitIdentForPath(identifier)
+	ns, tbl, err := r.splitIdentForPath(identifier)
 	if err != nil {
 		return err
 	}
@@ -1350,7 +1388,7 @@ func (r *Catalog) DropNamespace(ctx context.Context, namespace table.Identifier)
 		return err
 	}
 
-	path, err := endpointDeleteNamespace.reqPath(strings.Join(namespace, namespaceSeparator))
+	path, err := endpointDeleteNamespace.reqPath(r.encodeNamespace(namespace))
 	if err != nil {
 		return err
 	}
@@ -1398,7 +1436,7 @@ func (r *Catalog) listNamespacesPage(ctx context.Context, parent table.Identifie
 
 	v := url.Values{}
 	if len(parent) != 0 {
-		v.Set("parent", strings.Join(parent, namespaceSeparator))
+		v.Set("parent", r.namespaceToQueryParam(parent))
 	}
 	if pageSize > 0 {
 		v.Set("pageSize", strconv.Itoa(pageSize))
@@ -1437,7 +1475,7 @@ func (r *Catalog) LoadNamespaceProperties(ctx context.Context, namespace table.I
 		Props     iceberg.Properties `json:"properties"`
 	}
 
-	path, err := endpointLoadNamespace.reqPath(strings.Join(namespace, namespaceSeparator))
+	path, err := endpointLoadNamespace.reqPath(r.encodeNamespace(namespace))
 	if err != nil {
 		return nil, err
 	}
@@ -1467,7 +1505,7 @@ func (r *Catalog) UpdateNamespaceProperties(ctx context.Context, namespace table
 		Updates iceberg.Properties `json:"updates"`
 	}
 
-	ns := strings.Join(namespace, namespaceSeparator)
+	ns := r.encodeNamespace(namespace)
 
 	path, err := endpointUpdateNamespace.reqPath(ns)
 	if err != nil {
@@ -1498,7 +1536,7 @@ func (r *Catalog) CheckNamespaceExists(ctx context.Context, namespace table.Iden
 		return true, nil
 	}
 
-	path, err := endpointNamespaceExists.reqPath(strings.Join(namespace, namespaceSeparator))
+	path, err := endpointNamespaceExists.reqPath(r.encodeNamespace(namespace))
 	if err != nil {
 		return false, err
 	}
@@ -1517,7 +1555,7 @@ func (r *Catalog) CheckNamespaceExists(ctx context.Context, namespace table.Iden
 }
 
 func (r *Catalog) CheckTableExists(ctx context.Context, identifier table.Identifier) (bool, error) {
-	ns, tbl, err := splitIdentForPath(identifier)
+	ns, tbl, err := r.splitIdentForPath(identifier)
 	if err != nil {
 		return false, err
 	}
@@ -1590,7 +1628,7 @@ func (r *Catalog) listViewsPage(ctx context.Context, namespace table.Identifier,
 	if !r.endpoints.allowed(endpointListViews) {
 		return nil, "", nil
 	}
-	ns := strings.Join(namespace, namespaceSeparator)
+	ns := r.encodeNamespace(namespace)
 	path, err := endpointListViews.reqPath(ns)
 	if err != nil {
 		return nil, "", err
@@ -1641,7 +1679,7 @@ func (r *Catalog) DropView(ctx context.Context, identifier table.Identifier) err
 		return err
 	}
 
-	ns, view, err := splitIdentForPath(identifier)
+	ns, view, err := r.splitIdentForPath(identifier)
 	if err != nil {
 		return err
 	}
@@ -1658,7 +1696,7 @@ func (r *Catalog) DropView(ctx context.Context, identifier table.Identifier) err
 }
 
 func (r *Catalog) CheckViewExists(ctx context.Context, identifier table.Identifier) (bool, error) {
-	ns, view, err := splitIdentForPath(identifier)
+	ns, view, err := r.splitIdentForPath(identifier)
 	if err != nil {
 		return false, err
 	}
@@ -1728,7 +1766,7 @@ func (r *Catalog) CreateView(ctx context.Context, identifier table.Identifier, v
 		return nil, err
 	}
 
-	ns, viewName, err := splitIdentForPath(identifier)
+	ns, viewName, err := r.splitIdentForPath(identifier)
 	if err != nil {
 		return nil, err
 	}
@@ -1789,7 +1827,7 @@ func (r *Catalog) UpdateView(ctx context.Context, ident table.Identifier, requir
 		return nil, err
 	}
 
-	ns, viewName, err := splitIdentForPath(ident)
+	ns, viewName, err := r.splitIdentForPath(ident)
 	if err != nil {
 		return nil, err
 	}
@@ -1834,7 +1872,7 @@ func (r *Catalog) RegisterView(ctx context.Context, identifier table.Identifier,
 		return nil, err
 	}
 
-	ns, v, err := splitIdentForPath(identifier)
+	ns, v, err := r.splitIdentForPath(identifier)
 	if err != nil {
 		return nil, err
 	}
@@ -1871,7 +1909,7 @@ func (r *Catalog) LoadView(ctx context.Context, identifier table.Identifier) (*v
 		return nil, err
 	}
 
-	ns, v, err := splitIdentForPath(identifier)
+	ns, v, err := r.splitIdentForPath(identifier)
 	if err != nil {
 		return nil, err
 	}
diff --git a/catalog/rest/rest_internal_test.go b/catalog/rest/rest_internal_test.go
index cc89d36a..6330280e 100644
--- a/catalog/rest/rest_internal_test.go
+++ b/catalog/rest/rest_internal_test.go
@@ -982,3 +982,86 @@ func TestToPropsSigv4RegionFallback(t *testing.T) {
 		assert.False(t, ok)
 	})
 }
+
+func TestEncodeNamespace(t *testing.T) {
+	tests := []struct {
+		name          string
+		separator     string
+		namespace     []string
+		wantPath      string
+		wantQueryPart string
+	}{
+		{"default empty separator", "", []string{"a", "b"}, "a%1Fb", "a\x1fb"},
+		{"explicit unit separator", "%1F", []string{"a", "b"}, "a%1Fb", "a\x1fb"},
+		{"dot separator", "%2E", []string{"analytics", "prod"}, "analytics%2Eprod", "analytics.prod"},
+		{"single level", "%2E", []string{"db1"}, "db1", "db1"},
+		{"level needing escaping", "%2E", []string{"a/b", "c d"}, "a%2Fb%2Ec%20d", "a/b.c d"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := &Catalog{namespaceSeparator: tt.separator}
+			assert.Equal(t, tt.wantPath, r.encodeNamespace(tt.namespace))
+			assert.Equal(t, tt.wantQueryPart, r.namespaceToQueryParam(tt.namespace))
+		})
+	}
+}
+
+func TestNamespaceSeparatorFromConfig(t *testing.T) {
+	mux := http.NewServeMux()
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	mux.HandleFunc("/v1/config", func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(map[string]any{
+			"defaults":  map[string]any{},
+			"overrides": map[string]any{"namespace-separator": "%2E"},
+		})
+	})
+
+	var gotPath string
+	mux.HandleFunc("/v1/namespaces/", func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.EscapedPath()
+		json.NewEncoder(w).Encode(map[string]any{
+			"namespace":  []string{"analytics", "prod"},
+			"properties": map[string]any{"owner": "data-team"},
+		})
+	})
+
+	cat, err := NewCatalog(context.Background(), "rest", srv.URL)
+	require.NoError(t, err)
+	assert.Equal(t, "%2E", cat.namespaceSeparator)
+
+	props, err := cat.LoadNamespaceProperties(context.Background(), []string{"analytics", "prod"})
+	require.NoError(t, err)
+	assert.Equal(t, "/v1/namespaces/analytics%2Eprod", gotPath)
+	assert.Equal(t, "data-team", props["owner"])
+}
+
+func TestNamespaceSeparatorDefaultsToUnitSeparator(t *testing.T) {
+	mux := http.NewServeMux()
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	mux.HandleFunc("/v1/config", func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(map[string]any{
+			"defaults": map[string]any{}, "overrides": map[string]any{},
+		})
+	})
+
+	var gotPath string
+	mux.HandleFunc("/v1/namespaces/", func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.EscapedPath()
+		json.NewEncoder(w).Encode(map[string]any{
+			"namespace": []string{"a", "b"}, "properties": map[string]any{},
+		})
+	})
+
+	cat, err := NewCatalog(context.Background(), "rest", srv.URL)
+	require.NoError(t, err)
+	assert.Equal(t, "%1F", cat.namespaceSeparator)
+
+	_, err = cat.LoadNamespaceProperties(context.Background(), []string{"a", "b"})
+	require.NoError(t, err)
+	assert.Equal(t, "/v1/namespaces/a%1Fb", gotPath)
+}