You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by di...@apache.org on 2025/08/20 13:49:45 UTC
(polaris) branch paging-flag created (now 2e37f0e8e)
This is an automated email from the ASF dual-hosted git repository.
dimas pushed a change to branch paging-flag
in repository https://gitbox.apache.org/repos/asf/polaris.git
at 2e37f0e8e feat: enforce LIST_PAGINATION_ENABLED
This branch includes the following new commits:
new 2e37f0e8e feat: enforce LIST_PAGINATION_ENABLED
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(polaris) 01/01: feat: enforce LIST_PAGINATION_ENABLED
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimas pushed a commit to branch paging-flag
in repository https://gitbox.apache.org/repos/asf/polaris.git
commit 2e37f0e8edd7fb169197eb4a32b5ec044a96b171
Author: Dmitri Bourlatchkov <dm...@gmail.com>
AuthorDate: Tue Aug 19 17:52:32 2025 -0400
feat: enforce LIST_PAGINATION_ENABLED
The enforcement of the LIST_PAGINATION_ENABLED flag was missed in #1938.
This change make the flag effective as discussed in #2296.
Note: this causes a change in the default Polaris behaviour (no pagination
by default) with respect to the previous state of `main`. However, there
is no behaviour change with respect to 1.0.0 or 1.0.1 as previous releases
did not have #1938.
---
CHANGELOG.md | 3 ++
.../apache/polaris/service/it/env/CatalogApi.java | 14 +++++++
.../it/test/PolarisRestCatalogIntegrationBase.java | 48 ++++++++++++++++++++++
.../core/persistence/pagination/PageToken.java | 8 +++-
.../core/persistence/pagination/PageTokenUtil.java | 9 ++--
.../core/persistence/pagination/PageTokenTest.java | 18 ++++----
.../catalog/iceberg/IcebergCatalogHandler.java | 12 ++++--
.../catalog/AbstractIcebergCatalogTest.java | 2 +-
8 files changed, 97 insertions(+), 17 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 495a89b1a..88e730c4b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,9 @@ the authentication parameters are picked from the environment or configuration f
- The `DEFAULT_LOCATION_OBJECT_STORAGE_PREFIX_ENABLED` feature was added to support placing tables
at locations that better optimize for object storage.
+- The `LIST_PAGINATION_ENABLED` (default: false) feature flag can be used to enable pagination
+ in the Iceberg REST Catalog API.
+
- The Helm chart now supports Pod Disruption Budgets (PDBs) for Polaris components. This allows users to define
the minimum number of pods that must be available during voluntary disruptions, such as node maintenance.
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java
index eb400b1e6..d06e44209 100644
--- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java
+++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java
@@ -205,6 +205,20 @@ public class CatalogApi extends PolarisRestApi {
}
}
+ public ListTablesResponse listViews(
+ String catalog, Namespace namespace, String pageToken, String pageSize) {
+ String ns = RESTUtil.encodeNamespace(namespace);
+ Map<String, String> queryParams = new HashMap<>();
+ queryParams.put("pageToken", pageToken);
+ queryParams.put("pageSize", pageSize);
+ try (Response res =
+ request("v1/{cat}/namespaces/" + ns + "/views", Map.of("cat", catalog), queryParams)
+ .get()) {
+ assertThat(res.getStatus()).isEqualTo(Response.Status.OK.getStatusCode());
+ return res.readEntity(ListTablesResponse.class);
+ }
+ }
+
public void dropView(String catalog, TableIdentifier id) {
String ns = RESTUtil.encodeNamespace(id.namespace());
try (Response res =
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java
index ff0d9b7f1..f82619230 100644
--- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java
+++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java
@@ -2134,4 +2134,52 @@ public abstract class PolarisRestCatalogIntegrationBase extends CatalogTests<RES
restCatalog.dropNamespace(namespace);
}
}
+
+ @Test
+ public void testNonPaginatedListTablesViewNamespaces() {
+ Catalog catalog = managementApi.getCatalog(currentCatalogName);
+ Map<String, String> catalogProps = new HashMap<>(catalog.getProperties().toMap());
+ catalogProps.put(FeatureConfiguration.LIST_PAGINATION_ENABLED.catalogConfig(), "false");
+ managementApi.updateCatalog(catalog, catalogProps);
+
+ String prefix = "testNonPaginatedListTablesViewNamespaces";
+ Namespace namespace = Namespace.of(prefix);
+ restCatalog.createNamespace(namespace);
+ for (int i = 0; i < 5; i++) {
+ restCatalog.createNamespace(Namespace.of(prefix, "nested-ns" + i));
+ restCatalog.createTable(TableIdentifier.of(namespace, "table" + i), SCHEMA);
+ restCatalog
+ .buildView(TableIdentifier.of(namespace, "view" + i))
+ .withSchema(SCHEMA)
+ .withDefaultNamespace(namespace)
+ .withQuery("spark", VIEW_QUERY)
+ .create();
+ }
+
+ assertThat(catalogApi.listTables(currentCatalogName, namespace)).hasSize(5);
+ // Note: no pagination per feature config
+ ListTablesResponse response = catalogApi.listTables(currentCatalogName, namespace, null, "2");
+ assertThat(response.identifiers()).hasSize(5);
+ assertThat(response.nextPageToken()).isNull();
+ response = catalogApi.listTables(currentCatalogName, namespace, "fake-token", null);
+ assertThat(response.identifiers()).hasSize(5);
+ assertThat(response.nextPageToken()).isNull();
+
+ assertThat(catalogApi.listViews(currentCatalogName, namespace)).hasSize(5);
+ response = catalogApi.listViews(currentCatalogName, namespace, null, "2");
+ assertThat(response.identifiers()).hasSize(5);
+ assertThat(response.nextPageToken()).isNull();
+ response = catalogApi.listViews(currentCatalogName, namespace, "fake-token", null);
+ assertThat(response.identifiers()).hasSize(5);
+ assertThat(response.nextPageToken()).isNull();
+
+ assertThat(catalogApi.listNamespaces(currentCatalogName, namespace)).hasSize(5);
+ ListNamespacesResponse nsResponse =
+ catalogApi.listNamespaces(currentCatalogName, namespace, null, "2");
+ assertThat(nsResponse.namespaces()).hasSize(5);
+ assertThat(nsResponse.nextPageToken()).isNull();
+ nsResponse = catalogApi.listNamespaces(currentCatalogName, namespace, "fake-token", null);
+ assertThat(nsResponse.namespaces()).hasSize(5);
+ assertThat(nsResponse.nextPageToken()).isNull();
+ }
}
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/pagination/PageToken.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/pagination/PageToken.java
index a1dceffdd..6912de042 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/pagination/PageToken.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/pagination/PageToken.java
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import jakarta.annotation.Nullable;
import java.util.Optional;
import java.util.OptionalInt;
+import java.util.function.Supplier;
import org.apache.polaris.immutables.PolarisImmutable;
/** A wrapper for pagination information passed in as part of a request. */
@@ -86,7 +87,10 @@ public interface PageToken {
* @see Page#encodedResponseToken()
*/
static PageToken build(
- @Nullable String serializedPageToken, @Nullable Integer requestedPageSize) {
- return PageTokenUtil.decodePageRequest(serializedPageToken, requestedPageSize);
+ @Nullable String serializedPageToken,
+ @Nullable Integer requestedPageSize,
+ Supplier<Boolean> shouldDecodeToken) {
+ return PageTokenUtil.decodePageRequest(
+ serializedPageToken, requestedPageSize, shouldDecodeToken);
}
}
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/pagination/PageTokenUtil.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/pagination/PageTokenUtil.java
index 8a811f41c..ae0450a99 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/pagination/PageTokenUtil.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/pagination/PageTokenUtil.java
@@ -40,6 +40,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.ServiceLoader;
+import java.util.function.Supplier;
final class PageTokenUtil {
@@ -118,8 +119,10 @@ final class PageTokenUtil {
* token.
*/
static PageToken decodePageRequest(
- @Nullable String requestedPageToken, @Nullable Integer requestedPageSize) {
- if (requestedPageToken != null) {
+ @Nullable String requestedPageToken,
+ @Nullable Integer requestedPageSize,
+ Supplier<Boolean> shouldDecodeToken) {
+ if (requestedPageToken != null && shouldDecodeToken.get()) {
var bytes = Base64.getUrlDecoder().decode(requestedPageToken);
try {
var pageToken = SMILE_MAPPER.readValue(bytes, PageToken.class);
@@ -134,7 +137,7 @@ final class PageTokenUtil {
} catch (IOException e) {
throw new RuntimeException(e);
}
- } else if (requestedPageSize != null) {
+ } else if (requestedPageSize != null && shouldDecodeToken.get()) {
int pageSizeInt = requestedPageSize;
checkArgument(pageSizeInt >= 0, "Invalid page size");
return fromLimit(pageSizeInt);
diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/pagination/PageTokenTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/pagination/PageTokenTest.java
index 338bbc53f..dd5cb398c 100644
--- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/pagination/PageTokenTest.java
+++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/pagination/PageTokenTest.java
@@ -53,7 +53,7 @@ class PageTokenTest {
soft.assertThat(pageEverything.encodedResponseToken()).isNull();
soft.assertThat(pageEverything.items()).containsExactly(1, 2, 3, 4);
- r = PageToken.build(null, null);
+ r = PageToken.build(null, null, () -> true);
soft.assertThat(r.paginationRequested()).isFalse();
soft.assertThat(r.pageSize()).isEmpty();
soft.assertThat(r.value()).isEmpty();
@@ -62,7 +62,7 @@ class PageTokenTest {
@Test
public void testLimit() {
PageToken r = PageToken.fromLimit(123);
- soft.assertThat(r).isEqualTo(PageToken.build(null, 123));
+ soft.assertThat(r).isEqualTo(PageToken.build(null, 123, () -> true));
soft.assertThat(r.paginationRequested()).isTrue();
soft.assertThat(r.pageSize()).isEqualTo(OptionalInt.of(123));
soft.assertThat(r.value()).isEmpty();
@@ -71,7 +71,7 @@ class PageTokenTest {
@Test
public void testTokenValueForPaging() {
PageToken r = PageToken.fromLimit(2);
- soft.assertThat(r).isEqualTo(PageToken.build(null, 2));
+ soft.assertThat(r).isEqualTo(PageToken.build(null, 2, () -> true));
Page<Integer> pageMoreData =
Page.mapped(
r,
@@ -103,7 +103,7 @@ class PageTokenTest {
soft.assertThat(lastPageNotSaturated.items()).containsExactly(3);
r = PageToken.fromLimit(200);
- soft.assertThat(r).isEqualTo(PageToken.build(null, 200));
+ soft.assertThat(r).isEqualTo(PageToken.build(null, 200, () -> true));
Page<Integer> page200 =
Page.mapped(
r,
@@ -117,8 +117,10 @@ class PageTokenTest {
@ParameterizedTest
@MethodSource
public void testDeSer(Integer pageSize, String serializedPageToken, PageToken expectedPageToken) {
- soft.assertThat(PageTokenUtil.decodePageRequest(serializedPageToken, pageSize))
+ soft.assertThat(PageTokenUtil.decodePageRequest(serializedPageToken, pageSize, () -> true))
.isEqualTo(expectedPageToken);
+ soft.assertThat(PageTokenUtil.decodePageRequest(serializedPageToken, pageSize, () -> false))
+ .isEqualTo(PageToken.readEverything());
}
static Stream<Arguments> testDeSer() {
@@ -142,16 +144,16 @@ class PageTokenTest {
@ParameterizedTest
@MethodSource
public void testApiRoundTrip(Token token) {
- PageToken request = PageToken.build(null, 123);
+ PageToken request = PageToken.build(null, 123, () -> true);
Page<?> page = Page.mapped(request, Stream.of("i1"), Function.identity(), x -> token);
soft.assertThat(page.encodedResponseToken()).isNotBlank();
- PageToken r = PageToken.build(page.encodedResponseToken(), null);
+ PageToken r = PageToken.build(page.encodedResponseToken(), null, () -> true);
soft.assertThat(r.value()).contains(token);
soft.assertThat(r.paginationRequested()).isTrue();
soft.assertThat(r.pageSize()).isEqualTo(OptionalInt.of(123));
- r = PageToken.build(page.encodedResponseToken(), 456);
+ r = PageToken.build(page.encodedResponseToken(), 456, () -> true);
soft.assertThat(r.value()).contains(token);
soft.assertThat(r.paginationRequested()).isTrue();
soft.assertThat(r.pageSize()).isEqualTo(OptionalInt.of(456));
diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java
index b0cfc01b1..17cdd7af3 100644
--- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java
+++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java
@@ -18,6 +18,8 @@
*/
package org.apache.polaris.service.catalog.iceberg;
+import static org.apache.polaris.core.config.FeatureConfiguration.LIST_PAGINATION_ENABLED;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import io.smallrye.common.annotation.Identifier;
@@ -187,13 +189,17 @@ public class IcebergCatalogHandler extends CatalogHandler implements AutoCloseab
return isCreate;
}
+ private boolean shouldDecodeToken() {
+ return realmConfig.getConfig(LIST_PAGINATION_ENABLED, getResolvedCatalogEntity());
+ }
+
public ListNamespacesResponse listNamespaces(
Namespace parent, String pageToken, Integer pageSize) {
PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_NAMESPACES;
authorizeBasicNamespaceOperationOrThrow(op, parent);
if (baseCatalog instanceof IcebergCatalog polarisCatalog) {
- PageToken pageRequest = PageToken.build(pageToken, pageSize);
+ PageToken pageRequest = PageToken.build(pageToken, pageSize, this::shouldDecodeToken);
Page<Namespace> results = polarisCatalog.listNamespaces(parent, pageRequest);
return ListNamespacesResponse.builder()
.addAll(results.items())
@@ -332,7 +338,7 @@ public class IcebergCatalogHandler extends CatalogHandler implements AutoCloseab
authorizeBasicNamespaceOperationOrThrow(op, namespace);
if (baseCatalog instanceof IcebergCatalog polarisCatalog) {
- PageToken pageRequest = PageToken.build(pageToken, pageSize);
+ PageToken pageRequest = PageToken.build(pageToken, pageSize, this::shouldDecodeToken);
Page<TableIdentifier> results = polarisCatalog.listTables(namespace, pageRequest);
return ListTablesResponse.builder()
.addAll(results.items())
@@ -935,7 +941,7 @@ public class IcebergCatalogHandler extends CatalogHandler implements AutoCloseab
authorizeBasicNamespaceOperationOrThrow(op, namespace);
if (baseCatalog instanceof IcebergCatalog polarisCatalog) {
- PageToken pageRequest = PageToken.build(pageToken, pageSize);
+ PageToken pageRequest = PageToken.build(pageToken, pageSize, this::shouldDecodeToken);
Page<TableIdentifier> results = polarisCatalog.listViews(namespace, pageRequest);
return ListTablesResponse.builder()
.addAll(results.items())
diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/AbstractIcebergCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/AbstractIcebergCatalogTest.java
index 38c4db7e5..d10f82057 100644
--- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/AbstractIcebergCatalogTest.java
+++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/AbstractIcebergCatalogTest.java
@@ -2240,7 +2240,7 @@ public abstract class AbstractIcebergCatalogTest extends CatalogTests<IcebergCat
}
private static PageToken nextRequest(Page<?> previousPage) {
- return PageToken.build(previousPage.encodedResponseToken(), null);
+ return PageToken.build(previousPage.encodedResponseToken(), null, () -> true);
}
@Test