You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2025/03/04 15:22:28 UTC
(paimon) branch master updated: [core] RestCatalog: set up paged list tables/views/partitions (#5200)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 484fb1531b [core] RestCatalog: set up paged list tables/views/partitions (#5200)
484fb1531b is described below
commit 484fb1531b6090cab866ae8e277628cc8a7222bb
Author: XiaoHongbo <13...@qq.com>
AuthorDate: Tue Mar 4 23:22:22 2025 +0800
[core] RestCatalog: set up paged list tables/views/partitions (#5200)
---
.../src/main/java/org/apache/paimon/PagedList.java | 48 ++
.../org/apache/paimon/catalog/AbstractCatalog.java | 75 +++
.../java/org/apache/paimon/catalog/Catalog.java | 115 ++++
.../org/apache/paimon/catalog/DelegateCatalog.java | 36 ++
.../java/org/apache/paimon/rest/HttpClient.java | 36 +-
.../java/org/apache/paimon/rest/RESTCatalog.java | 287 +++++++++-
.../java/org/apache/paimon/rest/RESTClient.java | 7 +
.../java/org/apache/paimon/rest/ResourcePaths.java | 10 +
.../rest/responses/ListPartitionsResponse.java | 19 +-
...Response.java => ListTableDetailsResponse.java} | 38 +-
.../paimon/rest/responses/ListTablesResponse.java | 18 +-
...sResponse.java => ListViewDetailsResponse.java} | 38 +-
.../paimon/rest/responses/ListViewsResponse.java | 21 +-
.../org/apache/paimon/catalog/CatalogTestBase.java | 412 +++++++++++++
.../org/apache/paimon/rest/HttpClientTest.java | 24 +
.../org/apache/paimon/rest/RESTCatalogServer.java | 636 ++++++++++++++++++---
.../org/apache/paimon/rest/RESTCatalogTest.java | 465 +++++++++++++++
.../java/org/apache/paimon/hive/HiveCatalog.java | 31 +
paimon-open-api/rest-catalog-open-api.yaml | 131 +++++
.../paimon/open/api/RESTCatalogController.java | 96 +++-
20 files changed, 2410 insertions(+), 133 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/PagedList.java b/paimon-common/src/main/java/org/apache/paimon/PagedList.java
new file mode 100644
index 0000000000..77f78118ab
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/PagedList.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+
+/**
+ * Paged List which supports request data from page streaming.
+ *
+ * @since 1.1.0
+ */
+public class PagedList<T> {
+ private final List<T> elements;
+
+ @Nullable private final String nextPageToken;
+
+ public PagedList(List<T> elements, @Nullable String nextPageToken) {
+ this.elements = elements;
+ this.nextPageToken = nextPageToken;
+ }
+
+ public List<T> getElements() {
+ return this.elements;
+ }
+
+ @Nullable
+ public String getNextPageToken() {
+ return this.nextPageToken;
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
index 5a5b964393..b2ef6afd44 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
@@ -19,6 +19,7 @@
package org.apache.paimon.catalog;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.PagedList;
import org.apache.paimon.factories.FactoryUtil;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.FileStatus;
@@ -38,12 +39,16 @@ import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.table.system.SystemTableLoader;
import org.apache.paimon.types.RowType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
@@ -65,6 +70,8 @@ import static org.apache.paimon.utils.Preconditions.checkArgument;
/** Common implementation of {@link Catalog}. */
public abstract class AbstractCatalog implements Catalog {
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractCatalog.class);
+
protected final FileIO fileIO;
protected final Map<String, String> tableDefaultOptions;
protected final Options catalogOptions;
@@ -184,6 +191,13 @@ public abstract class AbstractCatalog implements Catalog {
return listPartitionsFromFileSystem(getTable(identifier));
}
+ @Override
+ public PagedList<Partition> listPartitionsPaged(
+ Identifier identifier, Integer maxResults, String pageToken)
+ throws TableNotExistException {
+ return new PagedList<>(listPartitions(identifier), null);
+ }
+
protected abstract void createDatabaseImpl(String name, Map<String, String> properties);
@Override
@@ -240,8 +254,69 @@ public abstract class AbstractCatalog implements Catalog {
return listTablesImpl(databaseName).stream().sorted().collect(Collectors.toList());
}
+ @Override
+ public PagedList<String> listTablesPaged(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ return new PagedList<>(listTables(databaseName), null);
+ }
+
+ @Override
+ public PagedList<Table> listTableDetailsPaged(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ if (isSystemDatabase(databaseName)) {
+ List<Table> systemTables =
+ SystemTableLoader.loadGlobalTableNames().stream()
+ .map(
+ tableName -> {
+ try {
+ return getTable(
+ Identifier.create(databaseName, tableName));
+ } catch (TableNotExistException ignored) {
+ LOG.warn(
+ "system table {}.{} does not exist",
+ databaseName,
+ tableName);
+ return null;
+ }
+ })
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList());
+ return new PagedList<>(systemTables, null);
+ }
+
+ // check db exists
+ getDatabase(databaseName);
+
+ return listTableDetailsPagedImpl(databaseName, maxResults, pageToken);
+ }
+
protected abstract List<String> listTablesImpl(String databaseName);
+ protected PagedList<Table> listTableDetailsPagedImpl(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ PagedList<String> pagedTableNames = listTablesPaged(databaseName, maxResults, pageToken);
+ return new PagedList<>(
+ pagedTableNames.getElements().stream()
+ .map(
+ tableName -> {
+ try {
+ return getTable(Identifier.create(databaseName, tableName));
+ } catch (TableNotExistException ignored) {
+ LOG.warn(
+ "table {}.{} does not exist",
+ databaseName,
+ tableName);
+ return null;
+ }
+ })
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList()),
+ pagedTableNames.getNextPageToken());
+ }
+
@Override
public void dropTable(Identifier identifier, boolean ignoreIfNotExists)
throws TableNotExistException {
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java
index e03cbc3ac8..5a5beb7117 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java
@@ -18,6 +18,7 @@
package org.apache.paimon.catalog;
+import org.apache.paimon.PagedList;
import org.apache.paimon.annotation.Public;
import org.apache.paimon.partition.Partition;
import org.apache.paimon.schema.Schema;
@@ -25,6 +26,8 @@ import org.apache.paimon.schema.SchemaChange;
import org.apache.paimon.table.Table;
import org.apache.paimon.view.View;
+import javax.annotation.Nullable;
+
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -130,6 +133,51 @@ public interface Catalog extends AutoCloseable {
*/
List<String> listTables(String databaseName) throws DatabaseNotExistException;
+ /**
+ * Get paged list names of tables under this database. An empty list is returned if none exists.
+ *
+ * <p>NOTE: Currently only RestCatalog will return pagedList data, other catalog will return all
+ * tables names like listTables.
+ *
+ * <p>NOTE: System tables will not be listed.
+ *
+ * @param databaseName Name of the database to list tables.
+ * @param maxResults Optional parameter indicating the maximum number of results to include in
+ * the result. If maxResults is not specified or set to 0, will return the default number of
+ * max results.
+ * @param pageToken Optional parameter indicating the next page token allows list to be start
+ * from a specific point.
+ * @return a list of the names of tables with provided page size in this database and next page
+ * token
+ * @throws DatabaseNotExistException if the database does not exist
+ */
+ PagedList<String> listTablesPaged(
+ String databaseName, @Nullable Integer maxResults, @Nullable String pageToken)
+ throws DatabaseNotExistException;
+
+ /**
+ * Get paged list of table details under this database. An empty list is returned if none
+ * exists.
+ *
+ * <p>NOTE: Currently only RestCatalog will return pagedList data, other catalog will return all
+ * table details, please use this method carefully for other catalog.
+ *
+ * <p>NOTE: System tables will not be listed.
+ *
+ * @param databaseName Name of the database to list table details.
+ * @param maxResults Optional parameter indicating the maximum number of results to include in
+ * the result. If maxResults is not specified or set to 0, will return the default number of
+ * max results.
+ * @param pageToken Optional parameter indicating the next page token allows list to be start
+ * from a specific point.
+ * @return a list of the table details with provided page size in this database and next page
+ * token
+ * @throws DatabaseNotExistException if the database does not exist
+ */
+ PagedList<Table> listTableDetailsPaged(
+ String databaseName, @Nullable Integer maxResults, @Nullable String pageToken)
+ throws DatabaseNotExistException;
+
/**
* Drop a table.
*
@@ -275,6 +323,26 @@ public interface Catalog extends AutoCloseable {
*/
List<Partition> listPartitions(Identifier identifier) throws TableNotExistException;
+ /**
+ * Get paged partitioned list of the table.
+ *
+ * <p>NOTE: Currently only RestCatalog will return pagedList data, other catalog will return all
+ * partitions.
+ *
+ * @param identifier path of the table to list partitions
+ * @param maxResults Optional parameter indicating the maximum number of results to include in
+ * the result. If maxResults is not specified or set to 0, will return the default number of
+ * max results.
+ * @param pageToken Optional parameter indicating the next page token allows list to be start
+ * from a specific point.
+ * @return a list of the partitions with provided page size(@param maxResults) in this table and
+ * next page token
+ * @throws TableNotExistException if the table does not exist
+ */
+ PagedList<Partition> listPartitionsPaged(
+ Identifier identifier, @Nullable Integer maxResults, @Nullable String pageToken)
+ throws TableNotExistException;
+
// ======================= view methods ===============================
/**
@@ -326,6 +394,53 @@ public interface Catalog extends AutoCloseable {
return Collections.emptyList();
}
+ /**
+ * Get paged list names of views under this database. An empty list is returned if none view
+ * exists.
+ *
+ * <p>NOTE: Currently only RestCatalog will return pagedList data, other catalog will return all
+ * view names, like listViews.
+ *
+ * @param databaseName Name of the database to list views.
+ * @param maxResults Optional parameter indicating the maximum number of results to include in
+ * the result. If maxResults is not specified or set to 0, will return the default number of
+ * max results.
+ * @param pageToken Optional parameter indicating the next page token allows list to be start
+ * from a specific point.
+ * @return a list of the names of views with provided page size in this database and next page
+ * token
+ * @throws DatabaseNotExistException if the database does not exist
+ */
+ default PagedList<String> listViewsPaged(
+ String databaseName, @Nullable Integer maxResults, @Nullable String pageToken)
+ throws DatabaseNotExistException {
+ return new PagedList<>(listViews(databaseName), null);
+ }
+
+ /**
+ * Get paged list view details under this database. An empty list is returned if none view
+ * exists.
+ *
+ * <p>NOTE: Currently only RestCatalog will return pagedList data, other catalog which supports
+ * view will return all view details, please use this method carefully for other catalog which
+ * supports view.
+ *
+ * @param databaseName Name of the database to list views.
+ * @param maxResults Optional parameter indicating the maximum number of results to include in
+ * the result. If maxResults is not specified or set to 0, will return the default number of
+ * max results.
+ * @param pageToken Optional parameter indicating the next page token allows list to be start
+ * from a specific point.
+ * @return a list of the view details with provided page size (@param maxResults) in this
+ * database and next page token
+ * @throws DatabaseNotExistException if the database does not exist
+ */
+ default PagedList<View> listViewDetailsPaged(
+ String databaseName, @Nullable Integer maxResults, @Nullable String pageToken)
+ throws DatabaseNotExistException {
+ return new PagedList<>(Collections.emptyList(), null);
+ }
+
/**
* Rename a view.
*
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java
index 2d8d3f3e12..36fc1243de 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java
@@ -18,6 +18,7 @@
package org.apache.paimon.catalog;
+import org.apache.paimon.PagedList;
import org.apache.paimon.partition.Partition;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
@@ -83,6 +84,20 @@ public abstract class DelegateCatalog implements Catalog {
return wrapped.listTables(databaseName);
}
+ @Override
+ public PagedList<String> listTablesPaged(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ return wrapped.listTablesPaged(databaseName, maxResults, pageToken);
+ }
+
+ @Override
+ public PagedList<Table> listTableDetailsPaged(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ return wrapped.listTableDetailsPaged(databaseName, maxResults, pageToken);
+ }
+
@Override
public void dropTable(Identifier identifier, boolean ignoreIfNotExists)
throws TableNotExistException {
@@ -159,6 +174,20 @@ public abstract class DelegateCatalog implements Catalog {
return wrapped.listViews(databaseName);
}
+ @Override
+ public PagedList<String> listViewsPaged(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ return wrapped.listViewsPaged(databaseName, maxResults, pageToken);
+ }
+
+ @Override
+ public PagedList<View> listViewDetailsPaged(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ return wrapped.listViewDetailsPaged(databaseName, maxResults, pageToken);
+ }
+
@Override
public void renameView(Identifier fromView, Identifier toView, boolean ignoreIfNotExists)
throws ViewNotExistException, ViewAlreadyExistException {
@@ -170,6 +199,13 @@ public abstract class DelegateCatalog implements Catalog {
return wrapped.listPartitions(identifier);
}
+ @Override
+ public PagedList<Partition> listPartitionsPaged(
+ Identifier identifier, Integer maxResults, String pageToken)
+ throws TableNotExistException {
+ return wrapped.listPartitionsPaged(identifier, maxResults, pageToken);
+ }
+
@Override
public void repairCatalog() {
wrapped.repairCatalog();
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java
index bc708d2d5b..4bad680f90 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java
@@ -29,6 +29,7 @@ import org.apache.paimon.utils.StringUtils;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
import okhttp3.Headers;
+import okhttp3.HttpUrl;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
@@ -41,6 +42,7 @@ import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
+import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -94,6 +96,22 @@ public class HttpClient implements RESTClient {
return exec(request, responseType);
}
+ @Override
+ public <T extends RESTResponse> T get(
+ String path,
+ Map<String, String> queryParams,
+ Class<T> responseType,
+ RESTAuthFunction restAuthFunction) {
+ Map<String, String> authHeaders = getHeaders(path, "GET", "", restAuthFunction);
+ Request request =
+ new Request.Builder()
+ .url(getRequestUrl(path, queryParams))
+ .get()
+ .headers(Headers.of(authHeaders))
+ .build();
+ return exec(request, responseType);
+ }
+
@Override
public <T extends RESTResponse> T post(
String path, RESTRequest body, RESTAuthFunction restAuthFunction) {
@@ -191,7 +209,23 @@ public class HttpClient implements RESTClient {
}
private String getRequestUrl(String path) {
- return StringUtils.isNullOrWhitespaceOnly(path) ? uri : uri + path;
+ return getRequestUrl(path, null);
+ }
+
+ private String getRequestUrl(String path, Map<String, String> queryParams) {
+ String fullPath = StringUtils.isNullOrWhitespaceOnly(path) ? uri : uri + path;
+ if (queryParams != null && !queryParams.isEmpty()) {
+ HttpUrl httpUrl = HttpUrl.parse(fullPath);
+ if (Objects.nonNull(httpUrl)) {
+ HttpUrl.Builder builder = httpUrl.newBuilder();
+ queryParams.forEach(builder::addQueryParameter);
+ return builder.build().toString();
+ } else {
+ return fullPath;
+ }
+ } else {
+ return fullPath;
+ }
}
private Map<String, String> getHeaders(
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
index 4fbf8dfa03..bd5a82497a 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
@@ -18,6 +18,7 @@
package org.apache.paimon.rest;
+import org.apache.paimon.PagedList;
import org.apache.paimon.Snapshot;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
@@ -68,7 +69,9 @@ import org.apache.paimon.rest.responses.GetViewResponse;
import org.apache.paimon.rest.responses.ListBranchesResponse;
import org.apache.paimon.rest.responses.ListDatabasesResponse;
import org.apache.paimon.rest.responses.ListPartitionsResponse;
+import org.apache.paimon.rest.responses.ListTableDetailsResponse;
import org.apache.paimon.rest.responses.ListTablesResponse;
+import org.apache.paimon.rest.responses.ListViewDetailsResponse;
import org.apache.paimon.rest.responses.ListViewsResponse;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
@@ -82,6 +85,11 @@ import org.apache.paimon.view.ViewImpl;
import org.apache.paimon.view.ViewSchema;
import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList;
+import org.apache.paimon.shade.guava30.com.google.common.collect.Maps;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
@@ -91,9 +99,11 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.stream.Collectors;
import static org.apache.paimon.CoreOptions.createCommitUser;
import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch;
@@ -111,7 +121,11 @@ import static org.apache.paimon.utils.ThreadPoolUtils.createScheduledThreadPool;
/** A catalog implementation for REST. */
public class RESTCatalog implements Catalog, SupportsSnapshots, SupportsBranches {
+ private static final Logger LOG = LoggerFactory.getLogger(RESTCatalog.class);
+
public static final String HEADER_PREFIX = "header.";
+ public static final String MAX_RESULTS = "maxResults";
+ public static final String PAGE_TOKEN = "pageToken";
private final RESTClient client;
private final ResourcePaths resourcePaths;
@@ -257,16 +271,101 @@ public class RESTCatalog implements Catalog, SupportsSnapshots, SupportsBranches
@Override
public List<String> listTables(String databaseName) throws DatabaseNotExistException {
+ try {
+ String pageToken = null;
+ Map<String, String> queryParams = Maps.newHashMap();
+ List<String> tables = new ArrayList<>();
+ do {
+ queryParams.put(PAGE_TOKEN, pageToken);
+ ListTablesResponse response =
+ client.get(
+ resourcePaths.tables(databaseName),
+ queryParams,
+ ListTablesResponse.class,
+ restAuthFunction);
+ if (Objects.nonNull(response)) {
+ pageToken = response.getNextPageToken();
+ tables.addAll(response.getTables());
+ } else {
+ LOG.warn(
+ "response of listTables for {} is null with params {}",
+ databaseName,
+ queryParams);
+ }
+ } while (StringUtils.isNotEmpty(pageToken));
+ return tables;
+ } catch (NoSuchResourceException e) {
+ throw new DatabaseNotExistException(databaseName);
+ }
+ }
+
+ @Override
+ public PagedList<String> listTablesPaged(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ Map<String, String> queryParams = buildPagedQueryParams(maxResults, pageToken);
try {
ListTablesResponse response =
client.get(
resourcePaths.tables(databaseName),
+ queryParams,
ListTablesResponse.class,
restAuthFunction);
- if (response.getTables() != null) {
- return response.getTables();
+ if (Objects.nonNull(response) && Objects.nonNull(response.getTables())) {
+ return new PagedList<>(response.getTables(), response.getNextPageToken());
+ } else {
+ LOG.warn(
+ "response of listTablesPaged for {} is null with maxResults {} and pageToken {}",
+ databaseName,
+ maxResults,
+ pageToken);
+ return new PagedList<>(Collections.emptyList(), null);
+ }
+ } catch (NoSuchResourceException e) {
+ throw new DatabaseNotExistException(databaseName);
+ }
+ }
+
+ @Override
+ public PagedList<Table> listTableDetailsPaged(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ Map<String, String> queryParams = buildPagedQueryParams(maxResults, pageToken);
+ try {
+ ListTableDetailsResponse response =
+ client.get(
+ resourcePaths.tableDetails(databaseName),
+ queryParams,
+ ListTableDetailsResponse.class,
+ restAuthFunction);
+ if (Objects.nonNull(response) && Objects.nonNull(response.getTableDetails())) {
+ return new PagedList<>(
+ response.getTableDetails().stream()
+ .map(
+ getTableResponse -> {
+ try {
+ return loadTableByResponse(
+ getTableResponse, databaseName);
+ } catch (TableNotExistException e) {
+ LOG.error(
+ "load table {}.{} by response {} failed",
+ databaseName,
+ getTableResponse.getName(),
+ getTableResponse,
+ e);
+ throw new RuntimeException(e);
+ }
+ })
+ .collect(Collectors.toList()),
+ response.getNextPageToken());
+ } else {
+ LOG.warn(
+ "response of listTableDetailsPaged for {} is null with maxResults {} and pageToken {}",
+ databaseName,
+ maxResults,
+ pageToken);
+ return new PagedList<>(Collections.emptyList(), null);
}
- return ImmutableList.of();
} catch (NoSuchResourceException e) {
throw new DatabaseNotExistException(databaseName);
}
@@ -560,16 +659,30 @@ public class RESTCatalog implements Catalog, SupportsSnapshots, SupportsBranches
@Override
public List<Partition> listPartitions(Identifier identifier) throws TableNotExistException {
try {
- ListPartitionsResponse response =
- client.get(
- resourcePaths.partitions(
- identifier.getDatabaseName(), identifier.getTableName()),
- ListPartitionsResponse.class,
- restAuthFunction);
- if (response == null || response.getPartitions() == null) {
- return Collections.emptyList();
- }
- return response.getPartitions();
+ String pageToken = null;
+ Map<String, String> queryParams = Maps.newHashMap();
+ List<Partition> partitions = new ArrayList<>();
+ do {
+ queryParams.put(PAGE_TOKEN, pageToken);
+ ListPartitionsResponse response =
+ client.get(
+ resourcePaths.partitions(
+ identifier.getDatabaseName(), identifier.getTableName()),
+ queryParams,
+ ListPartitionsResponse.class,
+ restAuthFunction);
+ if (Objects.nonNull(response)) {
+ pageToken = response.getNextPageToken();
+ partitions.addAll(response.getPartitions());
+ } else {
+ LOG.warn(
+ "response of listPartitions for {}.{} is null with params {}",
+ identifier.getDatabaseName(),
+ identifier.getTableName(),
+ queryParams);
+ }
+ } while (StringUtils.isNotEmpty(pageToken));
+ return partitions;
} catch (NoSuchResourceException e) {
throw new TableNotExistException(identifier);
} catch (ForbiddenException e) {
@@ -654,6 +767,35 @@ public class RESTCatalog implements Catalog, SupportsSnapshots, SupportsBranches
}
}
+ @Override
+ public PagedList<Partition> listPartitionsPaged(
+ Identifier identifier, Integer maxResults, String pageToken)
+ throws TableNotExistException {
+ Map<String, String> queryParams = buildPagedQueryParams(maxResults, pageToken);
+ try {
+ ListPartitionsResponse response =
+ client.get(
+ resourcePaths.partitions(
+ identifier.getDatabaseName(), identifier.getTableName()),
+ queryParams,
+ ListPartitionsResponse.class,
+ restAuthFunction);
+ if (Objects.nonNull(response)) {
+ return new PagedList<>(response.getPartitions(), response.getNextPageToken());
+ } else {
+ return new PagedList<>(Collections.emptyList(), null);
+ }
+
+ } catch (NoSuchResourceException e) {
+ throw new TableNotExistException(identifier);
+ } catch (ForbiddenException e) {
+ throw new TableNoPermissionException(identifier, e);
+ } catch (NotImplementedException e) {
+ // not a metastore partitioned table
+ return new PagedList<>(listPartitionsFromFileSystem(getTable(identifier)), null);
+ }
+ }
+
@Override
public View getView(Identifier identifier) throws ViewNotExistException {
try {
@@ -716,13 +858,98 @@ public class RESTCatalog implements Catalog, SupportsSnapshots, SupportsBranches
@Override
public List<String> listViews(String databaseName) throws DatabaseNotExistException {
+ try {
+ String pageToken = null;
+ Map<String, String> queryParams = Maps.newHashMap();
+ List<String> views = new ArrayList<>();
+ do {
+ queryParams.put(PAGE_TOKEN, pageToken);
+ ListViewsResponse response =
+ client.get(
+ resourcePaths.views(databaseName),
+ queryParams,
+ ListViewsResponse.class,
+ restAuthFunction);
+ if (Objects.nonNull(response)) {
+ pageToken = response.getNextPageToken();
+ views.addAll(response.getViews());
+ } else {
+ LOG.warn(
+ "response of listViews for {} is null with params {}",
+ databaseName,
+ queryParams);
+ }
+ } while (StringUtils.isNotEmpty(pageToken));
+ return views;
+ } catch (NoSuchResourceException e) {
+ throw new DatabaseNotExistException(databaseName);
+ }
+ }
+
+ @Override
+ public PagedList<String> listViewsPaged(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ Map<String, String> queryParams = buildPagedQueryParams(maxResults, pageToken);
try {
ListViewsResponse response =
client.get(
resourcePaths.views(databaseName),
+ queryParams,
ListViewsResponse.class,
restAuthFunction);
- return response.getViews();
+ if (Objects.nonNull(response) && Objects.nonNull(response.getViews())) {
+ return new PagedList<>(response.getViews(), response.getNextPageToken());
+ } else {
+ LOG.warn(
+ "response of listViewsPaged for {} is null with maxResults {} and pageToken {}",
+ databaseName,
+ maxResults,
+ pageToken);
+ return new PagedList<>(Collections.emptyList(), null);
+ }
+ } catch (NoSuchResourceException e) {
+ throw new DatabaseNotExistException(databaseName);
+ }
+ }
+
+ @Override
+ public PagedList<View> listViewDetailsPaged(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ Map<String, String> queryParams = buildPagedQueryParams(maxResults, pageToken);
+ try {
+ ListViewDetailsResponse response =
+ client.get(
+ resourcePaths.viewDetails(databaseName),
+ queryParams,
+ ListViewDetailsResponse.class,
+ restAuthFunction);
+ if (Objects.nonNull(response) && Objects.nonNull(response.getViewDetails())) {
+ return new PagedList<>(
+ response.getViewDetails().stream()
+ .map(
+ viewResponse -> {
+ ViewSchema schema = viewResponse.getSchema();
+ return new ViewImpl(
+ Identifier.create(
+ databaseName, viewResponse.getName()),
+ schema.fields(),
+ schema.query(),
+ schema.dialects(),
+ schema.comment(),
+ schema.options());
+ })
+ .collect(Collectors.toList()),
+ response.getNextPageToken());
+ } else {
+ LOG.warn(
+ "response of listViewDetailsPaged for {} is null with maxResults {} and pageToken {}",
+ databaseName,
+ maxResults,
+ pageToken);
+ return new PagedList<>(Collections.emptyList(), null);
+ }
} catch (NoSuchResourceException e) {
throw new DatabaseNotExistException(databaseName);
}
@@ -777,4 +1004,36 @@ public class RESTCatalog implements Catalog, SupportsSnapshots, SupportsBranches
return refreshExecutor;
}
+
+ private Table loadTableByResponse(GetTableResponse getTableResponse, String databaseName)
+ throws TableNotExistException {
+ Identifier identifier = Identifier.create(databaseName, getTableResponse.getName());
+ TableMetadata.Loader tableMetaDataLoader =
+ tableIdentifier -> {
+ TableSchema schema =
+ TableSchema.create(
+ getTableResponse.getSchemaId(), getTableResponse.getSchema());
+ return new TableMetadata(
+ schema, getTableResponse.isExternal(), getTableResponse.getId());
+ };
+ return CatalogUtils.loadTable(
+ this,
+ identifier,
+ path -> fileIOForData(path, identifier),
+ this::fileIOFromOptions,
+ tableMetaDataLoader,
+ null,
+ null);
+ }
+
+ private Map<String, String> buildPagedQueryParams(Integer maxResults, String pageToken) {
+ Map<String, String> queryParams = Maps.newHashMap();
+ if (Objects.nonNull(maxResults) && maxResults > 0) {
+ queryParams.put(MAX_RESULTS, maxResults.toString());
+ }
+ if (Objects.nonNull(pageToken)) {
+ queryParams.put(PAGE_TOKEN, pageToken);
+ }
+ return queryParams;
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java
index 7446836653..ac1a27a2be 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java
@@ -21,6 +21,7 @@ package org.apache.paimon.rest;
import org.apache.paimon.rest.auth.RESTAuthFunction;
import java.io.Closeable;
+import java.util.Map;
/** Interface for a basic HTTP Client for interfacing with the REST catalog. */
public interface RESTClient extends Closeable {
@@ -28,6 +29,12 @@ public interface RESTClient extends Closeable {
<T extends RESTResponse> T get(
String path, Class<T> responseType, RESTAuthFunction restAuthFunction);
+ <T extends RESTResponse> T get(
+ String path,
+ Map<String, String> queryParams,
+ Class<T> responseType,
+ RESTAuthFunction restAuthFunction);
+
<T extends RESTResponse> T post(
String path, RESTRequest body, RESTAuthFunction restAuthFunction);
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java
index e733654901..7499ad1625 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java
@@ -32,6 +32,8 @@ public class ResourcePaths {
private static final String PARTITIONS = "partitions";
private static final String BRANCHES = "branches";
private static final String VIEWS = "views";
+ private static final String TABLE_DETAILS = "table-details";
+ private static final String VIEW_DETAILS = "view-details";
public static final String QUERY_PARAMETER_WAREHOUSE_KEY = "warehouse";
public static String config(String warehouse) {
@@ -64,6 +66,10 @@ public class ResourcePaths {
return SLASH.join(V1, prefix, DATABASES, databaseName, TABLES);
}
+ public String tableDetails(String databaseName) {
+ return SLASH.join(V1, prefix, DATABASES, databaseName, TABLE_DETAILS);
+ }
+
public String table(String databaseName, String tableName) {
return SLASH.join(V1, prefix, DATABASES, databaseName, TABLES, tableName);
}
@@ -121,6 +127,10 @@ public class ResourcePaths {
return SLASH.join(V1, prefix, DATABASES, databaseName, VIEWS);
}
+ public String viewDetails(String databaseName) {
+ return SLASH.join(V1, prefix, DATABASES, databaseName, VIEW_DETAILS);
+ }
+
public String view(String databaseName, String viewName) {
return SLASH.join(V1, prefix, DATABASES, databaseName, VIEWS, viewName);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java
index 9a3ea8d4ee..8c91d88ca6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java
@@ -34,16 +34,33 @@ public class ListPartitionsResponse implements RESTResponse {
public static final String FIELD_PARTITIONS = "partitions";
+ private static final String FIELD_NEXT_PAGE_TOKEN = "nextPageToken";
+
@JsonProperty(FIELD_PARTITIONS)
private final List<Partition> partitions;
- @JsonCreator
+ @JsonProperty(FIELD_NEXT_PAGE_TOKEN)
+ private final String nextPageToken;
+
public ListPartitionsResponse(@JsonProperty(FIELD_PARTITIONS) List<Partition> partitions) {
+ this(partitions, null);
+ }
+
+ @JsonCreator
+ public ListPartitionsResponse(
+ @JsonProperty(FIELD_PARTITIONS) List<Partition> partitions,
+ @JsonProperty(FIELD_NEXT_PAGE_TOKEN) String nextPageToken) {
this.partitions = partitions;
+ this.nextPageToken = nextPageToken;
}
@JsonGetter(FIELD_PARTITIONS)
public List<Partition> getPartitions() {
return partitions;
}
+
+ @JsonGetter(FIELD_NEXT_PAGE_TOKEN)
+ public String getNextPageToken() {
+ return this.nextPageToken;
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListTableDetailsResponse.java
similarity index 54%
copy from paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java
copy to paimon-core/src/main/java/org/apache/paimon/rest/responses/ListTableDetailsResponse.java
index 9a3ea8d4ee..84f55e842f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListTableDetailsResponse.java
@@ -18,7 +18,6 @@
package org.apache.paimon.rest.responses;
-import org.apache.paimon.partition.Partition;
import org.apache.paimon.rest.RESTResponse;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
@@ -28,22 +27,39 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonPro
import java.util.List;
-/** Response for listing partitions. */
+/** Response for listing table details. */
@JsonIgnoreProperties(ignoreUnknown = true)
-public class ListPartitionsResponse implements RESTResponse {
+public class ListTableDetailsResponse implements RESTResponse {
- public static final String FIELD_PARTITIONS = "partitions";
+ private static final String FIELD_TABLE_DETAILS = "tableDetails";
+ private static final String FIELD_NEXT_PAGE_TOKEN = "nextPageToken";
- @JsonProperty(FIELD_PARTITIONS)
- private final List<Partition> partitions;
+ @JsonProperty(FIELD_TABLE_DETAILS)
+ private final List<GetTableResponse> tableDetails;
+
+ @JsonProperty(FIELD_NEXT_PAGE_TOKEN)
+ private final String nextPageToken;
+
+ public ListTableDetailsResponse(
+ @JsonProperty(FIELD_TABLE_DETAILS) List<GetTableResponse> tableDetails) {
+ this(tableDetails, null);
+ }
@JsonCreator
- public ListPartitionsResponse(@JsonProperty(FIELD_PARTITIONS) List<Partition> partitions) {
- this.partitions = partitions;
+ public ListTableDetailsResponse(
+ @JsonProperty(FIELD_TABLE_DETAILS) List<GetTableResponse> tableDetails,
+ @JsonProperty(FIELD_NEXT_PAGE_TOKEN) String nextPageToken) {
+ this.tableDetails = tableDetails;
+ this.nextPageToken = nextPageToken;
+ }
+
+ @JsonGetter(FIELD_TABLE_DETAILS)
+ public List<GetTableResponse> getTableDetails() {
+ return this.tableDetails;
}
- @JsonGetter(FIELD_PARTITIONS)
- public List<Partition> getPartitions() {
- return partitions;
+ @JsonGetter(FIELD_NEXT_PAGE_TOKEN)
+ public String getNextPageToken() {
+ return this.nextPageToken;
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListTablesResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListTablesResponse.java
index 9e1ce5ae87..419ea91484 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListTablesResponse.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListTablesResponse.java
@@ -32,17 +32,33 @@ import java.util.List;
public class ListTablesResponse implements RESTResponse {
private static final String FIELD_TABLES = "tables";
+ private static final String FIELD_NEXT_PAGE_TOKEN = "nextPageToken";
@JsonProperty(FIELD_TABLES)
private final List<String> tables;
- @JsonCreator
+ @JsonProperty(FIELD_NEXT_PAGE_TOKEN)
+ private final String nextPageToken;
+
public ListTablesResponse(@JsonProperty(FIELD_TABLES) List<String> tables) {
+ this(tables, null);
+ }
+
+ @JsonCreator
+ public ListTablesResponse(
+ @JsonProperty(FIELD_TABLES) List<String> tables,
+ @JsonProperty(FIELD_NEXT_PAGE_TOKEN) String nextPageToken) {
this.tables = tables;
+ this.nextPageToken = nextPageToken;
}
@JsonGetter(FIELD_TABLES)
public List<String> getTables() {
return this.tables;
}
+
+ @JsonGetter(FIELD_NEXT_PAGE_TOKEN)
+ public String getNextPageToken() {
+ return this.nextPageToken;
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListViewDetailsResponse.java
similarity index 55%
copy from paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java
copy to paimon-core/src/main/java/org/apache/paimon/rest/responses/ListViewDetailsResponse.java
index 9a3ea8d4ee..3a080f1f25 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListViewDetailsResponse.java
@@ -18,7 +18,6 @@
package org.apache.paimon.rest.responses;
-import org.apache.paimon.partition.Partition;
import org.apache.paimon.rest.RESTResponse;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
@@ -28,22 +27,39 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonPro
import java.util.List;
-/** Response for listing partitions. */
+/** Response for listing view details. */
@JsonIgnoreProperties(ignoreUnknown = true)
-public class ListPartitionsResponse implements RESTResponse {
+public class ListViewDetailsResponse implements RESTResponse {
- public static final String FIELD_PARTITIONS = "partitions";
+ private static final String FIELD_VIEW_DETAILS = "viewDetails";
+ private static final String FIELD_NEXT_PAGE_TOKEN = "nextPageToken";
- @JsonProperty(FIELD_PARTITIONS)
- private final List<Partition> partitions;
+ @JsonProperty(FIELD_VIEW_DETAILS)
+ private final List<GetViewResponse> viewDetails;
+
+ @JsonProperty(FIELD_NEXT_PAGE_TOKEN)
+ private final String nextPageToken;
+
+ public ListViewDetailsResponse(
+ @JsonProperty(FIELD_VIEW_DETAILS) List<GetViewResponse> viewDetails) {
+ this(viewDetails, null);
+ }
@JsonCreator
- public ListPartitionsResponse(@JsonProperty(FIELD_PARTITIONS) List<Partition> partitions) {
- this.partitions = partitions;
+ public ListViewDetailsResponse(
+ @JsonProperty(FIELD_VIEW_DETAILS) List<GetViewResponse> viewDetails,
+ @JsonProperty(FIELD_NEXT_PAGE_TOKEN) String nextPageToken) {
+ this.viewDetails = viewDetails;
+ this.nextPageToken = nextPageToken;
+ }
+
+ @JsonGetter(FIELD_VIEW_DETAILS)
+ public List<GetViewResponse> getViewDetails() {
+ return this.viewDetails;
}
- @JsonGetter(FIELD_PARTITIONS)
- public List<Partition> getPartitions() {
- return partitions;
+ @JsonGetter(FIELD_NEXT_PAGE_TOKEN)
+ public String getNextPageToken() {
+ return this.nextPageToken;
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListViewsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListViewsResponse.java
index d785fd68c2..851412ca5f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListViewsResponse.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListViewsResponse.java
@@ -27,22 +27,39 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonPro
import java.util.List;
-/** Response for listing tables. */
+/** Response for listing views. */
@JsonIgnoreProperties(ignoreUnknown = true)
public class ListViewsResponse implements RESTResponse {
private static final String FIELD_VIEWS = "views";
+ private static final String FIELD_NEXT_PAGE_TOKEN = "nextPageToken";
+
@JsonProperty(FIELD_VIEWS)
private final List<String> views;
- @JsonCreator
+ @JsonProperty(FIELD_NEXT_PAGE_TOKEN)
+ private final String nextPageToken;
+
public ListViewsResponse(@JsonProperty(FIELD_VIEWS) List<String> views) {
+ this(views, null);
+ }
+
+ @JsonCreator
+ public ListViewsResponse(
+ @JsonProperty(FIELD_VIEWS) List<String> views,
+ @JsonProperty(FIELD_NEXT_PAGE_TOKEN) String nextPageToken) {
this.views = views;
+ this.nextPageToken = nextPageToken;
}
@JsonGetter(FIELD_VIEWS)
public List<String> getViews() {
return this.views;
}
+
+ @JsonGetter(FIELD_NEXT_PAGE_TOKEN)
+ public String getNextPageToken() {
+ return this.nextPageToken;
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
index 833b0ed5a8..1013a562e4 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
@@ -19,6 +19,7 @@
package org.apache.paimon.catalog;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.PagedList;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.ResolvingFileIO;
import org.apache.paimon.options.CatalogOptions;
@@ -26,6 +27,7 @@ import org.apache.paimon.options.Options;
import org.apache.paimon.partition.Partition;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
+import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.FormatTable;
import org.apache.paimon.table.Table;
@@ -64,6 +66,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
/** Base test class of paimon catalog in {@link Catalog}. */
public abstract class CatalogTestBase {
@@ -259,6 +262,102 @@ public abstract class CatalogTestBase {
.isThrownBy(() -> catalog.listTables("non_existing_db"));
}
+ @Test
+ public void testListTablesPaged() throws Exception {
+ // List tables paged returns an empty list when there are no tables in the database
+ String databaseName = "tables_paged_db";
+ catalog.createDatabase(databaseName, false);
+ PagedList<String> pagedTables = catalog.listTablesPaged(databaseName, null, null);
+ assertThat(pagedTables.getElements()).isEmpty();
+ assertNull(pagedTables.getNextPageToken());
+
+ String[] tableNames = {"table1", "table2", "table3", "abd", "def", "opr"};
+ for (String tableName : tableNames) {
+ catalog.createTable(
+ Identifier.create(databaseName, tableName), DEFAULT_TABLE_SCHEMA, false);
+ }
+
+ // List tables paged returns a list with the names of all tables in the database in all
+ // catalogs except RestCatalog
+ // even if the maxResults or pageToken is not null
+ pagedTables = catalog.listTablesPaged(databaseName, null, null);
+ assertPagedTables(pagedTables, tableNames);
+
+ int maxResults = 2;
+ pagedTables = catalog.listTablesPaged(databaseName, maxResults, null);
+ assertPagedTables(pagedTables, tableNames);
+
+ String pageToken = "table1";
+ pagedTables = catalog.listTablesPaged(databaseName, maxResults, pageToken);
+ assertPagedTables(pagedTables, tableNames);
+
+ maxResults = 8;
+ pagedTables = catalog.listTablesPaged(databaseName, maxResults, null);
+ assertPagedTables(pagedTables, tableNames);
+
+ pagedTables = catalog.listTablesPaged(databaseName, maxResults, pageToken);
+ assertPagedTables(pagedTables, tableNames);
+
+ // List tables throws DatabaseNotExistException when the database does not exist
+ final int finalMaxResults = maxResults;
+ assertThatExceptionOfType(Catalog.DatabaseNotExistException.class)
+ .isThrownBy(
+ () ->
+ catalog.listTablesPaged(
+ "non_existing_db", finalMaxResults, pageToken));
+ }
+
+ @Test
+ public void testListTableDetailsPaged() throws Exception {
+ // List table details returns an empty list when there are no tables in the database
+ String databaseName = "table_details_paged_db";
+ catalog.createDatabase(databaseName, false);
+ PagedList<Table> pagedTableDetails =
+ catalog.listTableDetailsPaged(databaseName, null, null);
+ assertThat(pagedTableDetails.getElements()).isEmpty();
+ assertNull(pagedTableDetails.getNextPageToken());
+
+ // List table details paged returns a list with all table in the database in all catalogs
+ // except RestCatalog
+ // even if the maxResults or pageToken is not null
+ String[] tableNames = {"table1", "table2", "table3", "abd", "def", "opr"};
+ for (String tableName : tableNames) {
+ catalog.createTable(
+ Identifier.create(databaseName, tableName), DEFAULT_TABLE_SCHEMA, false);
+ }
+
+ pagedTableDetails = catalog.listTableDetailsPaged(databaseName, null, null);
+ assertPagedTableDetails(pagedTableDetails, tableNames.length, tableNames);
+ assertNull(pagedTableDetails.getNextPageToken());
+
+ int maxResults = 2;
+ pagedTableDetails = catalog.listTableDetailsPaged(databaseName, maxResults, null);
+ assertPagedTableDetails(pagedTableDetails, tableNames.length, tableNames);
+ assertNull(pagedTableDetails.getNextPageToken());
+
+ String pageToken = "table1";
+ pagedTableDetails = catalog.listTableDetailsPaged(databaseName, maxResults, pageToken);
+ assertPagedTableDetails(pagedTableDetails, tableNames.length, tableNames);
+ assertNull(pagedTableDetails.getNextPageToken());
+
+ maxResults = 8;
+ pagedTableDetails = catalog.listTableDetailsPaged(databaseName, maxResults, null);
+ assertPagedTableDetails(pagedTableDetails, tableNames.length, tableNames);
+ assertNull(pagedTableDetails.getNextPageToken());
+
+ pagedTableDetails = catalog.listTableDetailsPaged(databaseName, maxResults, pageToken);
+ assertPagedTableDetails(pagedTableDetails, tableNames.length, tableNames);
+ assertNull(pagedTableDetails.getNextPageToken());
+
+ // List table details throws DatabaseNotExistException when the database does not exist
+ final int finalMaxResults = maxResults;
+ assertThatExceptionOfType(Catalog.DatabaseNotExistException.class)
+ .isThrownBy(
+ () ->
+ catalog.listTableDetailsPaged(
+ "non_existing_db", finalMaxResults, pageToken));
+ }
+
@Test
public void testCreateTable() throws Exception {
catalog.createDatabase("test_db", false);
@@ -984,6 +1083,110 @@ public abstract class CatalogTestBase {
.isInstanceOf(Catalog.ViewNotExistException.class);
}
+ @Test
+ public void testListViewsPaged() throws Exception {
+ if (!supportsView()) {
+ return;
+ }
+
+ // List views returns an empty list when there are no views in the database
+ String databaseName = "views_paged_db";
+ catalog.createDatabase(databaseName, false);
+ PagedList<String> pagedViews = catalog.listViewsPaged(databaseName, null, null);
+ assertThat(pagedViews.getElements()).isEmpty();
+ assertNull(pagedViews.getNextPageToken());
+
+ // List views paged returns a list with the names of all views in the database in all
+ // catalogs except RestCatalog
+ // even if the maxResults or pageToken is not null
+ View view = buildView(databaseName);
+ String[] viewNames = {"view1", "view2", "view3", "abd", "def", "opr"};
+ for (String viewName : viewNames) {
+ catalog.createView(Identifier.create(databaseName, viewName), view, false);
+ }
+
+ pagedViews = catalog.listViewsPaged(databaseName, null, null);
+ assertPagedViews(pagedViews, viewNames);
+
+ int maxResults = 2;
+ pagedViews = catalog.listViewsPaged(databaseName, maxResults, null);
+ assertPagedViews(pagedViews, viewNames);
+
+ String pageToken = "view1";
+ pagedViews = catalog.listViewsPaged(databaseName, maxResults, pageToken);
+ assertPagedViews(pagedViews, viewNames);
+
+ maxResults = 8;
+ pagedViews = catalog.listViewsPaged(databaseName, maxResults, null);
+ assertPagedViews(pagedViews, viewNames);
+
+ pagedViews = catalog.listViewsPaged(databaseName, maxResults, pageToken);
+ assertPagedViews(pagedViews, viewNames);
+
+ // List views throws DatabaseNotExistException when the database does not exist
+ final int finalMaxResults = maxResults;
+ assertThatExceptionOfType(Catalog.DatabaseNotExistException.class)
+ .isThrownBy(
+ () ->
+ catalog.listViewsPaged(
+ "non_existing_db", finalMaxResults, pageToken));
+ }
+
+ @Test
+ public void testListViewDetailsPaged() throws Exception {
+ if (!supportsView()) {
+ return;
+ }
+
+ // List views returns an empty list when there are no views in the database
+ String databaseName = "view_details_paged_db";
+ catalog.createDatabase(databaseName, false);
+ PagedList<View> pagedViewDetailsPaged =
+ catalog.listViewDetailsPaged(databaseName, null, null);
+ assertThat(pagedViewDetailsPaged.getElements()).isEmpty();
+ assertNull(pagedViewDetailsPaged.getNextPageToken());
+
+ // List view details paged returns a list with all view in the database in all catalogs
+ // except RestCatalog
+ // even if the maxResults or pageToken is not null
+ View view = buildView(databaseName);
+ String[] viewNames = {"view1", "view2", "view3", "abd", "def", "opr"};
+ for (String viewName : viewNames) {
+ catalog.createView(Identifier.create(databaseName, viewName), view, false);
+ }
+
+ pagedViewDetailsPaged = catalog.listViewDetailsPaged(databaseName, null, null);
+ assertPagedViewDetails(pagedViewDetailsPaged, view, viewNames.length, viewNames);
+ assertNull(pagedViewDetailsPaged.getNextPageToken());
+
+ int maxResults = 2;
+ pagedViewDetailsPaged = catalog.listViewDetailsPaged(databaseName, maxResults, null);
+ assertPagedViewDetails(pagedViewDetailsPaged, view, viewNames.length, viewNames);
+ assertNull(pagedViewDetailsPaged.getNextPageToken());
+
+ String pageToken = "view1";
+ pagedViewDetailsPaged = catalog.listViewDetailsPaged(databaseName, maxResults, pageToken);
+ assertPagedViewDetails(pagedViewDetailsPaged, view, viewNames.length, viewNames);
+ assertNull(pagedViewDetailsPaged.getNextPageToken());
+
+ maxResults = 8;
+ pagedViewDetailsPaged = catalog.listViewDetailsPaged(databaseName, maxResults, null);
+ assertPagedViewDetails(pagedViewDetailsPaged, view, viewNames.length, viewNames);
+ assertNull(pagedViewDetailsPaged.getNextPageToken());
+
+ pagedViewDetailsPaged = catalog.listViewDetailsPaged(databaseName, maxResults, pageToken);
+ assertPagedViewDetails(pagedViewDetailsPaged, view, viewNames.length, viewNames);
+ assertNull(pagedViewDetailsPaged.getNextPageToken());
+
+ // List view details throws DatabaseNotExistException when the database does not exist
+ final int finalMaxResults = maxResults;
+ assertThatExceptionOfType(Catalog.DatabaseNotExistException.class)
+ .isThrownBy(
+ () ->
+ catalog.listViewDetailsPaged(
+ "non_existing_db", finalMaxResults, pageToken));
+ }
+
@Test
public void testFormatTable() throws Exception {
if (!supportsFormatTable()) {
@@ -1091,6 +1294,69 @@ public abstract class CatalogTestBase {
partitionSpecs));
}
+ @Test
+ public void testListPartitionsPaged() throws Exception {
+ if (!supportPartitions()) {
+ return;
+ }
+ String databaseName = "partitions_paged_db";
+ List<Map<String, String>> partitionSpecs =
+ Arrays.asList(
+ Collections.singletonMap("dt", "20250101"),
+ Collections.singletonMap("dt", "20250102"),
+ Collections.singletonMap("dt", "20240102"),
+ Collections.singletonMap("dt", "20260101"),
+ Collections.singletonMap("dt", "20250104"),
+ Collections.singletonMap("dt", "20250103"));
+ catalog.dropDatabase(databaseName, true, true);
+ catalog.createDatabase(databaseName, true);
+ Identifier identifier = Identifier.create(databaseName, "table");
+ catalog.createTable(
+ identifier,
+ Schema.newBuilder()
+ .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(METASTORE_TAG_TO_PARTITION.key(), "dt")
+ .column("col", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .partitionKeys("dt")
+ .build(),
+ true);
+
+ catalog.createPartitions(identifier, partitionSpecs);
+
+ // List partitions paged returns a list with all partitions of the table in all catalogs
+ // except RestCatalog even
+ // if the maxResults or pageToken is not null
+ PagedList<Partition> pagedPartitions = catalog.listPartitionsPaged(identifier, null, null);
+ Map[] specs = partitionSpecs.toArray(new Map[0]);
+ assertPagedPartitions(pagedPartitions, specs.length, specs);
+
+ int maxResults = 2;
+ pagedPartitions = catalog.listPartitionsPaged(identifier, maxResults, null);
+ assertPagedPartitions(pagedPartitions, specs.length, specs);
+
+ String pageToken = "dt=20250101";
+ pagedPartitions = catalog.listPartitionsPaged(identifier, maxResults, pageToken);
+ assertPagedPartitions(pagedPartitions, specs.length, specs);
+
+ maxResults = 8;
+ pagedPartitions = catalog.listPartitionsPaged(identifier, maxResults, null);
+ assertPagedPartitions(pagedPartitions, specs.length, specs);
+
+ pagedPartitions = catalog.listPartitionsPaged(identifier, maxResults, pageToken);
+ assertPagedPartitions(pagedPartitions, specs.length, specs);
+
+ // List partitions throws TableNotExistException when the table does not exist
+ final int finalMaxResults = maxResults;
+ assertThatExceptionOfType(Catalog.TableNotExistException.class)
+ .isThrownBy(
+ () ->
+ catalog.listPartitionsPaged(
+ Identifier.create(databaseName, "non_existing_table"),
+ finalMaxResults,
+ pageToken));
+ }
+
@Test
public void testAlterPartitions() throws Exception {
if (!supportPartitions()) {
@@ -1157,4 +1423,150 @@ public abstract class CatalogTestBase {
protected boolean supportPartitions() {
return false;
}
+
+ protected boolean supportPagedList() {
+ return false;
+ }
+
+ private void assertPagedTables(PagedList<String> tablePagedList, String... tableNames) {
+ List<String> tables = tablePagedList.getElements();
+ if (supportPagedList()) {
+ assertThat(tables).containsExactly(tableNames);
+ } else {
+ assertThat(tables).containsExactlyInAnyOrder(tableNames);
+ }
+ assertNull(tablePagedList.getNextPageToken());
+ }
+
+ protected void assertPagedTableDetails(
+ PagedList<Table> tableDetailsPagedList, int size, String... tableNames) {
+ List<Table> tableDetails = tableDetailsPagedList.getElements();
+ assertEquals(size, tableDetails.size());
+ if (supportPagedList()) {
+ assertThat(tableDetails.stream().map(Table::name).collect(Collectors.toList()))
+ .containsExactly(tableNames);
+ } else {
+ assertThat(tableDetails.stream().map(Table::name).collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(tableNames);
+ }
+ List<Schema> schemas =
+ tableDetails.stream()
+ .filter(table -> table instanceof FileStoreTable)
+ .map(table -> (FileStoreTable) table)
+ .map(FileStoreTable::schema)
+ .map(TableSchema::toSchema)
+ .collect(Collectors.toList());
+ assertThat(
+ schemas.stream()
+ .map(Schema::fields)
+ .allMatch(fields -> DEFAULT_TABLE_SCHEMA.fields().equals(fields)))
+ .isTrue();
+ assertThat(
+ schemas.stream()
+ .map(Schema::partitionKeys)
+ .allMatch(
+ partitionKeys ->
+ DEFAULT_TABLE_SCHEMA
+ .partitionKeys()
+ .equals(partitionKeys)))
+ .isTrue();
+ assertThat(
+ schemas.stream()
+ .map(Schema::primaryKeys)
+ .allMatch(
+ primaryKeys ->
+ DEFAULT_TABLE_SCHEMA
+ .primaryKeys()
+ .equals(primaryKeys)))
+ .isTrue();
+ assertThat(
+ schemas.stream()
+ .map(Schema::options)
+ .allMatch(
+ options ->
+ DEFAULT_TABLE_SCHEMA.options().size()
+ <= options.size()))
+ .isTrue(); // output schema has path
+ assertThat(
+ schemas.stream()
+ .map(Schema::comment)
+ .allMatch(
+ comment -> DEFAULT_TABLE_SCHEMA.comment().equals(comment)))
+ .isTrue();
+ }
+
+ protected View buildView(String databaseName) {
+ Identifier identifier = new Identifier(databaseName, "my_view");
+ RowType rowType = RowType.builder().field("str", DataTypes.STRING()).build();
+ String query = "SELECT * FROM OTHER_TABLE";
+
+ Map<String, String> dialects = new HashMap<>();
+ if (supportsViewDialects()) {
+ dialects.put("spark", "SELECT * FROM SPARK_TABLE");
+ }
+ return new ViewImpl(
+ identifier, rowType.getFields(), query, dialects, null, new HashMap<>());
+ }
+
+ protected void assertPagedViews(PagedList<String> viewPagedList, String... viewNames) {
+ List<String> views = viewPagedList.getElements();
+ if (supportPagedList()) {
+ assertThat(views).containsExactly(viewNames);
+ } else {
+ assertThat(views).containsExactlyInAnyOrder(viewNames);
+ }
+ }
+
+ protected void assertPagedViewDetails(
+ PagedList<View> viewDetailsPagedList, View view, int size, String... viewNames) {
+ List<View> viewDetails = viewDetailsPagedList.getElements();
+ assertEquals(size, viewDetails.size());
+ if (supportPagedList()) {
+ assertThat(viewDetails.stream().map(View::name).collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(viewNames);
+ } else {
+ assertThat(viewDetails.stream().map(View::name).collect(Collectors.toList()))
+ .containsExactlyInAnyOrder(viewNames);
+ }
+ assertThat(
+ viewDetails.stream()
+ .map(View::rowType)
+ .allMatch(rowType -> view.rowType().equals(rowType)))
+ .isTrue();
+ assertThat(
+ viewDetails.stream()
+ .map(View::query)
+ .allMatch(query -> view.query().equals(query)))
+ .isTrue();
+ assertThat(
+ viewDetails.stream()
+ .map(View::dialects)
+ .allMatch(dialects -> view.dialects().equals(dialects)))
+ .isTrue();
+ assertThat(
+ viewDetails.stream()
+ .map(View::comment)
+ .allMatch(comment -> view.comment().equals(comment)))
+ .isTrue();
+ assertThat(
+ viewDetails.stream()
+ .map(View::options)
+ .allMatch(options -> view.options().size() <= options.size()))
+ .isTrue(); // last ddl time
+ }
+
+ @SafeVarargs
+ protected final void assertPagedPartitions(
+ PagedList<Partition> partitionsPagedList,
+ int size,
+ Map<String, String>... partitionSpecs) {
+ List<Partition> partitions = partitionsPagedList.getElements();
+ assertEquals(size, partitions.size());
+ if (supportPagedList()) {
+ assertThat(partitions.stream().map(Partition::spec)).containsExactly(partitionSpecs);
+ } else {
+ assertThat(partitions.stream().map(Partition::spec))
+ .containsExactlyInAnyOrder(partitionSpecs);
+ }
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java
index 9fdbd529ea..beaf8b7078 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java
@@ -87,6 +87,30 @@ public class HttpClientTest {
assertEquals(mockResponseData.data(), response.data());
}
+ @Test
+ public void testGetSuccessWithQueryParams() {
+ server.enqueueResponse(mockResponseDataStr, 200);
+ Map<String, String> queryParams = ImmutableMap.of("maxResults", "10", "pageToken", "abc");
+ MockRESTData response =
+ httpClient.get(MOCK_PATH, queryParams, MockRESTData.class, restAuthFunction);
+ assertEquals(mockResponseData.data(), response.data());
+
+ server.enqueueResponse(mockResponseDataStr, 200);
+ queryParams = ImmutableMap.of("pageToken", "abc");
+ response = httpClient.get(MOCK_PATH, queryParams, MockRESTData.class, restAuthFunction);
+ assertEquals(mockResponseData.data(), response.data());
+
+ server.enqueueResponse(mockResponseDataStr, 200);
+ queryParams = ImmutableMap.of("maxResults", "10");
+ response = httpClient.get(MOCK_PATH, queryParams, MockRESTData.class, restAuthFunction);
+ assertEquals(mockResponseData.data(), response.data());
+
+ server.enqueueResponse(mockResponseDataStr, 200);
+ queryParams = new HashMap<>();
+ response = httpClient.get(MOCK_PATH, queryParams, MockRESTData.class, restAuthFunction);
+ assertEquals(mockResponseData.data(), response.data());
+ }
+
@Test
public void testGetFail() {
server.enqueueResponse(errorResponseStr, 400);
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
index 4f96d0727d..cce4a94df9 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
@@ -66,7 +66,9 @@ import org.apache.paimon.rest.responses.GetViewResponse;
import org.apache.paimon.rest.responses.ListBranchesResponse;
import org.apache.paimon.rest.responses.ListDatabasesResponse;
import org.apache.paimon.rest.responses.ListPartitionsResponse;
+import org.apache.paimon.rest.responses.ListTableDetailsResponse;
import org.apache.paimon.rest.responses.ListTablesResponse;
+import org.apache.paimon.rest.responses.ListViewDetailsResponse;
import org.apache.paimon.rest.responses.ListViewsResponse;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
@@ -86,15 +88,19 @@ import okhttp3.mockwebserver.Dispatcher;
import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
import okhttp3.mockwebserver.RecordedRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
@@ -107,6 +113,11 @@ import static org.apache.paimon.rest.RESTObjectMapper.OBJECT_MAPPER;
/** Mock REST server for testing. */
public class RESTCatalogServer {
+ private static final Logger LOG = LoggerFactory.getLogger(RESTCatalogServer.class);
+
+ public static final int DEFAULT_MAX_RESULTS = 100;
+ public static final String MAX_RESULTS = "maxResults";
+ public static final String PAGE_TOKEN = "pageToken";
private final String prefix;
private final String databaseUri;
@@ -206,8 +217,13 @@ public class RESTCatalogServer {
if (!databaseStore.containsKey(databaseName)) {
throw new Catalog.DatabaseNotExistException(databaseName);
}
- boolean isViews = resources.length == 2 && "views".equals(resources[1]);
- boolean isTables = resources.length == 2 && "tables".equals(resources[1]);
+ boolean isViews = resources.length == 2 && resources[1].startsWith("views");
+ boolean isViewsDetails =
+ resources.length == 2 && resources[1].startsWith("view-details");
+ boolean isTables =
+ resources.length == 2 && resources[1].startsWith("tables");
+ boolean isTableDetails =
+ resources.length == 2 && resources[1].startsWith("table-details");
boolean isTableRename =
resources.length == 3
&& "tables".equals(resources[1])
@@ -240,7 +256,7 @@ public class RESTCatalogServer {
boolean isPartitions =
resources.length == 4
&& "tables".equals(resources[1])
- && "partitions".equals(resources[3]);
+ && resources[3].startsWith("partitions");
boolean isDropPartitions =
resources.length == 5
@@ -340,8 +356,12 @@ public class RESTCatalogServer {
return tableHandle(request, identifier);
} else if (isTables) {
return tablesHandle(request, databaseName);
+ } else if (isTableDetails) {
+ return tableDetailsHandle(request, databaseName);
} else if (isViews) {
return viewsHandle(request, databaseName);
+ } else if (isViewsDetails) {
+ return viewDetailsHandle(request, databaseName);
} else if (isViewRename) {
return renameViewHandle(request);
} else if (isView) {
@@ -598,49 +618,206 @@ public class RESTCatalogServer {
private MockResponse tablesHandle(RecordedRequest request, String databaseName)
throws Exception {
- RESTResponse response;
if (databaseStore.containsKey(databaseName)) {
- switch (request.getMethod()) {
- case "GET":
- List<String> tables = new ArrayList<>();
- for (Map.Entry<String, TableMetadata> entry : tableMetadataStore.entrySet()) {
- Identifier identifier = Identifier.fromString(entry.getKey());
- if (databaseName.equals(identifier.getDatabaseName())) {
- tables.add(identifier.getTableName());
+ if (Objects.nonNull(request)
+ && Objects.nonNull(request.getMethod())
+ && Objects.nonNull(request.getRequestUrl())) {
+ switch (request.getMethod()) {
+ case "GET":
+ List<String> tables = listTables(databaseName);
+ return generateFinalListTablesResponse(request, tables);
+ case "POST":
+ CreateTableRequest requestBody =
+ OBJECT_MAPPER.readValue(
+ request.getBody().readUtf8(), CreateTableRequest.class);
+ Identifier identifier = requestBody.getIdentifier();
+ Schema schema = requestBody.getSchema();
+ TableMetadata tableMetadata;
+ if (isFormatTable(schema)) {
+ tableMetadata = createFormatTable(identifier, schema);
+ } else {
+ catalog.createTable(identifier, schema, false);
+ tableMetadata =
+ createTableMetadata(
+ requestBody.getIdentifier(),
+ 1L,
+ requestBody.getSchema(),
+ UUID.randomUUID().toString(),
+ false);
}
- }
- response = new ListTablesResponse(tables);
- return mockResponse(response, 200);
- case "POST":
- CreateTableRequest requestBody =
- OBJECT_MAPPER.readValue(
- request.getBody().readUtf8(), CreateTableRequest.class);
- Identifier identifier = requestBody.getIdentifier();
- Schema schema = requestBody.getSchema();
- TableMetadata tableMetadata;
- if (isFormatTable(schema)) {
- tableMetadata = createFormatTable(identifier, schema);
- } else {
- catalog.createTable(identifier, schema, false);
- tableMetadata =
- createTableMetadata(
- requestBody.getIdentifier(),
- 1L,
- requestBody.getSchema(),
- UUID.randomUUID().toString(),
- false);
- }
- tableMetadataStore.put(
- requestBody.getIdentifier().getFullName(), tableMetadata);
- return new MockResponse().setResponseCode(200);
- default:
- return new MockResponse().setResponseCode(404);
+ tableMetadataStore.put(
+ requestBody.getIdentifier().getFullName(), tableMetadata);
+ return new MockResponse().setResponseCode(200);
+ default:
+ return new MockResponse().setResponseCode(404);
+ }
+ } else {
+ return mockResponse(
+ new ErrorResponse(
+ ErrorResponseResourceType.TABLE,
+ null,
+ "invalid input " + request,
+ 400),
+ 400);
}
}
return mockResponse(
new ErrorResponse(ErrorResponseResourceType.DATABASE, null, "", 404), 404);
}
+ private List<String> listTables(String databaseName) {
+ List<String> tables = new ArrayList<>();
+ for (Map.Entry<String, TableMetadata> entry : tableMetadataStore.entrySet()) {
+ Identifier identifier = Identifier.fromString(entry.getKey());
+ if (databaseName.equals(identifier.getDatabaseName())) {
+ tables.add(identifier.getTableName());
+ }
+ }
+ return tables;
+ }
+
+ private MockResponse generateFinalListTablesResponse(
+ RecordedRequest request, List<String> tables) {
+ RESTResponse response;
+ if (!tables.isEmpty()) {
+ int maxResults;
+ try {
+ maxResults = getMaxResults(request);
+ } catch (Exception e) {
+ LOG.error(
+ "parse maxResults {} to int failed",
+ Objects.nonNull(request.getRequestUrl())
+ ? request.getRequestUrl().queryParameter(MAX_RESULTS)
+ : null);
+ return mockResponse(
+ new ErrorResponse(
+ ErrorResponseResourceType.TABLE,
+ null,
+ "invalid input queryParameter maxResults"
+ + request.getRequestUrl().queryParameter(MAX_RESULTS),
+ 400),
+ 400);
+ }
+ String pageToken =
+ Objects.nonNull(request.getRequestUrl())
+ ? request.getRequestUrl().queryParameter(PAGE_TOKEN)
+ : null;
+
+ List<String> sortedTables =
+ tables.stream().sorted(String::compareTo).collect(Collectors.toList());
+ if (maxResults > sortedTables.size() && pageToken == null) {
+ response = new ListTablesResponse(sortedTables, null);
+ } else {
+ response = generateListTablesResponse(sortedTables, maxResults, pageToken);
+ }
+ } else {
+ response = new ListTablesResponse(new ArrayList<>(), null);
+ }
+ return mockResponse(response, 200);
+ }
+
+ private ListTablesResponse generateListTablesResponse(
+ List<String> sortedTables, int maxResults, String pageToken) {
+ List<String> pagedTables = new ArrayList<>();
+ for (String sortedTable : sortedTables) {
+ if (pagedTables.size() < maxResults) {
+ if (pageToken == null) {
+ pagedTables.add(sortedTable);
+ } else if (sortedTable.compareTo(pageToken) > 0) {
+ pagedTables.add(sortedTable);
+ }
+ } else {
+ break;
+ }
+ }
+ String nextPageToken = getNextPageTokenForNames(pagedTables, maxResults);
+ return new ListTablesResponse(pagedTables, nextPageToken);
+ }
+
+ private MockResponse tableDetailsHandle(RecordedRequest request, String databaseName) {
+ RESTResponse response;
+ if (Objects.nonNull(request)
+ && "GET".equals(request.getMethod())
+ && Objects.nonNull(request.getRequestUrl())) {
+
+ List<GetTableResponse> tableDetails = listTableDetails(databaseName);
+ if (!tableDetails.isEmpty()) {
+ int maxResults;
+ try {
+ maxResults = getMaxResults(request);
+ } catch (Exception e) {
+ LOG.error(
+ "parse maxResults {} to int failed",
+ Objects.nonNull(request.getRequestUrl())
+ ? request.getRequestUrl().queryParameter(MAX_RESULTS)
+ : null);
+ return mockResponse(
+ new ErrorResponse(
+ ErrorResponseResourceType.TABLE,
+ null,
+ "invalid input queryParameter maxResults"
+ + request.getRequestUrl().queryParameter(MAX_RESULTS),
+ 400),
+ 400);
+ }
+ String pageToken = request.getRequestUrl().queryParameter(PAGE_TOKEN);
+ List<GetTableResponse> sortedTableDetails =
+ tableDetails.stream()
+ .sorted(Comparator.comparing(GetTableResponse::getName))
+ .collect(Collectors.toList());
+
+ if (maxResults > sortedTableDetails.size() && pageToken == null) {
+ response = new ListTableDetailsResponse(sortedTableDetails, null);
+ } else {
+ response =
+ generateListTableDetailsResponse(
+ sortedTableDetails, maxResults, pageToken);
+ }
+ } else {
+ response = new ListTableDetailsResponse(Collections.emptyList(), null);
+ }
+ return mockResponse(response, 200);
+ } else {
+ return new MockResponse().setResponseCode(404);
+ }
+ }
+
+ private List<GetTableResponse> listTableDetails(String databaseName) {
+ List<GetTableResponse> tableDetails = new ArrayList<>();
+ for (Map.Entry<String, TableMetadata> entry : tableMetadataStore.entrySet()) {
+ Identifier identifier = Identifier.fromString(entry.getKey());
+ if (databaseName.equals(identifier.getDatabaseName())) {
+ GetTableResponse getTableResponse =
+ new GetTableResponse(
+ entry.getValue().uuid(),
+ identifier.getTableName(),
+ entry.getValue().isExternal(),
+ entry.getValue().schema().id(),
+ entry.getValue().schema().toSchema());
+ tableDetails.add(getTableResponse);
+ }
+ }
+ return tableDetails;
+ }
+
+ private ListTableDetailsResponse generateListTableDetailsResponse(
+ List<GetTableResponse> sortedTableDetails, int maxResults, String pageToken) {
+ List<GetTableResponse> pagedTableDetails = new ArrayList<>();
+ for (GetTableResponse getTableResponse : sortedTableDetails) {
+ if (pagedTableDetails.size() < maxResults) {
+ if (pageToken == null) {
+ pagedTableDetails.add(getTableResponse);
+ } else if (getTableResponse.getName().compareTo(pageToken) > 0) {
+ pagedTableDetails.add(getTableResponse);
+ }
+ } else {
+ break;
+ }
+ }
+ String nextPageToken = getNextPageTokenForTables(pagedTableDetails, maxResults);
+ return new ListTableDetailsResponse(pagedTableDetails, nextPageToken);
+ }
+
private boolean isFormatTable(Schema schema) {
return Options.fromMap(schema.options()).get(TYPE) == FORMAT_TABLE;
}
@@ -702,67 +879,292 @@ public class RESTCatalogServer {
private MockResponse partitionsApiHandle(RecordedRequest request, Identifier tableIdentifier)
throws Exception {
+ if (Objects.nonNull(request)
+ && Objects.nonNull(request.getMethod())
+ && Objects.nonNull(request.getRequestUrl())) {
+ switch (request.getMethod()) {
+ case "GET":
+ List<Partition> partitions =
+ tablePartitionsStore.get(tableIdentifier.getFullName());
+ return generateFinalListPartitionsResponse(request, partitions);
+ case "POST":
+ CreatePartitionsRequest requestBody =
+ OBJECT_MAPPER.readValue(
+ request.getBody().readUtf8(), CreatePartitionsRequest.class);
+ tablePartitionsStore.put(
+ tableIdentifier.getFullName(),
+ requestBody.getPartitionSpecs().stream()
+ .map(partition -> spec2Partition(partition))
+ .collect(Collectors.toList()));
+ return new MockResponse().setResponseCode(200);
+ default:
+ return new MockResponse().setResponseCode(404);
+ }
+ } else {
+ return mockResponse(
+ new ErrorResponse(
+ ErrorResponseResourceType.TABLE, null, "invalid input " + request, 400),
+ 400);
+ }
+ }
+
+ private MockResponse generateFinalListPartitionsResponse(
+ RecordedRequest request, List<Partition> partitions) {
RESTResponse response;
- switch (request.getMethod()) {
- case "GET":
- List<Partition> partitions =
- tablePartitionsStore.get(tableIdentifier.getFullName());
- response = new ListPartitionsResponse(partitions);
- return mockResponse(response, 200);
- case "POST":
- CreatePartitionsRequest requestBody =
- OBJECT_MAPPER.readValue(
- request.getBody().readUtf8(), CreatePartitionsRequest.class);
- tablePartitionsStore.put(
- tableIdentifier.getFullName(),
- requestBody.getPartitionSpecs().stream()
- .map(partition -> spec2Partition(partition))
- .collect(Collectors.toList()));
- return new MockResponse().setResponseCode(200);
- default:
- return new MockResponse().setResponseCode(404);
+ if (Objects.nonNull(partitions) && !partitions.isEmpty()) {
+ int maxResults;
+ try {
+ maxResults = getMaxResults(request);
+ } catch (Exception e) {
+ LOG.error(
+ "parse maxResults {} to int failed",
+ Objects.nonNull(request.getRequestUrl())
+ ? request.getRequestUrl().queryParameter(MAX_RESULTS)
+ : null);
+ return mockResponse(
+ new ErrorResponse(
+ ErrorResponseResourceType.TABLE,
+ null,
+ "invalid input queryParameter maxResults"
+ + request.getRequestUrl().queryParameter(MAX_RESULTS),
+ 400),
+ 400);
+ }
+ String pageToken =
+ Objects.nonNull(request.getRequestUrl())
+ ? request.getRequestUrl().queryParameter(PAGE_TOKEN)
+ : null;
+
+ List<Partition> sortedPartitions =
+ partitions.stream()
+ .sorted(Comparator.comparing(this::getPartitionSortKey))
+ .collect(Collectors.toList());
+
+ if ((maxResults > sortedPartitions.size()) && pageToken == null) {
+ response = new ListPartitionsResponse(sortedPartitions, null);
+ } else {
+ response = generateListPartitionsResponse(sortedPartitions, maxResults, pageToken);
+ }
+ } else {
+ response = new ListPartitionsResponse(Collections.emptyList(), null);
}
+ return mockResponse(response, 200);
+ }
+
+ private ListPartitionsResponse generateListPartitionsResponse(
+ List<Partition> sortedPartitions, int maxResults, String pageToken) {
+ List<Partition> pagedPartitions = new ArrayList<>();
+ for (Partition partition : sortedPartitions) {
+ if (pagedPartitions.size() < maxResults) {
+ if (pageToken == null) {
+ pagedPartitions.add(partition);
+ } else if (getPartitionSortKey(partition).compareTo(pageToken) > 0) {
+ pagedPartitions.add(partition);
+ }
+ } else {
+ break;
+ }
+ }
+ String nextPageToken = getNextPageTokenForPartitions(pagedPartitions, maxResults);
+ return new ListPartitionsResponse(pagedPartitions, nextPageToken);
}
private MockResponse viewsHandle(RecordedRequest request, String databaseName)
throws Exception {
+ if (Objects.nonNull(request)
+ && Objects.nonNull(request.getMethod())
+ && Objects.nonNull(request.getRequestUrl())) {
+ switch (request.getMethod()) {
+ case "GET":
+ List<String> views = listViews(databaseName);
+ return generateFinalListViewsResponse(request, views);
+ case "POST":
+ CreateViewRequest requestBody =
+ OBJECT_MAPPER.readValue(
+ request.getBody().readUtf8(), CreateViewRequest.class);
+ Identifier identifier = requestBody.getIdentifier();
+ ViewSchema schema = requestBody.getSchema();
+ ViewImpl view =
+ new ViewImpl(
+ requestBody.getIdentifier(),
+ schema.fields(),
+ schema.query(),
+ schema.dialects(),
+ schema.comment(),
+ schema.options());
+ if (viewStore.containsKey(identifier.getFullName())) {
+ throw new Catalog.ViewAlreadyExistException(identifier);
+ }
+ viewStore.put(identifier.getFullName(), view);
+ return new MockResponse().setResponseCode(200);
+ default:
+ return new MockResponse().setResponseCode(404);
+ }
+ } else {
+ return mockResponse(
+ new ErrorResponse(ErrorResponseResourceType.TABLE, null, "invalid input", 400),
+ 400);
+ }
+ }
+
+ private List<String> listViews(String databaseName) {
+ return viewStore.keySet().stream()
+ .map(Identifier::fromString)
+ .filter(identifier -> identifier.getDatabaseName().equals(databaseName))
+ .map(Identifier::getTableName)
+ .collect(Collectors.toList());
+ }
+
+ private MockResponse generateFinalListViewsResponse(
+ RecordedRequest request, List<String> views) {
RESTResponse response;
- switch (request.getMethod()) {
- case "GET":
- List<String> views =
- viewStore.keySet().stream()
- .map(Identifier::fromString)
- .filter(
- identifier ->
- identifier.getDatabaseName().equals(databaseName))
- .map(Identifier::getTableName)
+ if (!views.isEmpty()) {
+ int maxResults;
+ try {
+ maxResults = getMaxResults(request);
+ } catch (Exception e) {
+ LOG.error(
+ "parse maxResults {} to int failed",
+ Objects.nonNull(request.getRequestUrl())
+ ? request.getRequestUrl().queryParameter(MAX_RESULTS)
+ : null);
+ return mockResponse(
+ new ErrorResponse(
+ ErrorResponseResourceType.TABLE,
+ null,
+ "invalid input queryParameter maxResults"
+ + request.getRequestUrl().queryParameter(MAX_RESULTS),
+ 400),
+ 400);
+ }
+ String pageToken =
+ Objects.nonNull(request.getRequestUrl())
+ ? request.getRequestUrl().queryParameter(PAGE_TOKEN)
+ : null;
+
+ List<String> sortedViews =
+ views.stream().sorted(String::compareTo).collect(Collectors.toList());
+
+ if ((maxResults > sortedViews.size()) && pageToken == null) {
+ response = new ListViewsResponse(sortedViews, null);
+ } else {
+ response = generateListViewsResponse(sortedViews, maxResults, pageToken);
+ }
+ } else {
+ response = new ListViewsResponse(Collections.emptyList(), null);
+ }
+ return mockResponse(response, 200);
+ }
+
+ private ListViewsResponse generateListViewsResponse(
+ List<String> sortedViews, int maxResults, String pageToken) {
+ List<String> pagedViews = new ArrayList<>();
+ for (String view : sortedViews) {
+ if (pagedViews.size() < maxResults) {
+ if (pageToken == null) {
+ pagedViews.add(view);
+ } else if (view.compareTo(pageToken) > 0) {
+ pagedViews.add(view);
+ }
+ } else {
+ break;
+ }
+ }
+ String nextPageToken = getNextPageTokenForNames(pagedViews, maxResults);
+ return new ListViewsResponse(pagedViews, nextPageToken);
+ }
+
+ private MockResponse viewDetailsHandle(RecordedRequest request, String databaseName) {
+ RESTResponse response;
+ if (Objects.nonNull(request)
+ && "GET".equals(request.getMethod())
+ && Objects.nonNull(request.getRequestUrl())) {
+
+ List<GetViewResponse> viewDetails = listViewDetails(databaseName);
+ if (!viewDetails.isEmpty()) {
+
+ int maxResults;
+ try {
+ maxResults = getMaxResults(request);
+ } catch (Exception e) {
+ LOG.error(
+ "parse maxResults {} to int failed",
+ Objects.nonNull(request.getRequestUrl())
+ ? request.getRequestUrl().queryParameter(MAX_RESULTS)
+ : null);
+ return mockResponse(
+ new ErrorResponse(
+ ErrorResponseResourceType.TABLE,
+ null,
+ "invalid input queryParameter maxResults"
+ + request.getRequestUrl().queryParameter(MAX_RESULTS),
+ 400),
+ 400);
+ }
+ String pageToken =
+ Objects.nonNull(request.getRequestUrl())
+ ? request.getRequestUrl().queryParameter(PAGE_TOKEN)
+ : null;
+
+ List<GetViewResponse> sortedViewDetails =
+ viewDetails.stream()
+ .sorted(Comparator.comparing(GetViewResponse::getName))
.collect(Collectors.toList());
- response = new ListViewsResponse(views);
- return mockResponse(response, 200);
- case "POST":
- CreateViewRequest requestBody =
- OBJECT_MAPPER.readValue(
- request.getBody().readUtf8(), CreateViewRequest.class);
- Identifier identifier = requestBody.getIdentifier();
- ViewSchema schema = requestBody.getSchema();
- ViewImpl view =
- new ViewImpl(
- requestBody.getIdentifier(),
- schema.fields(),
- schema.query(),
- schema.dialects(),
- schema.comment(),
- schema.options());
- if (viewStore.containsKey(identifier.getFullName())) {
- throw new Catalog.ViewAlreadyExistException(identifier);
+
+ if ((maxResults > sortedViewDetails.size()) && pageToken == null) {
+ response = new ListViewDetailsResponse(sortedViewDetails, null);
+ } else {
+ response =
+ generateListViewDetailsResponse(
+ sortedViewDetails, maxResults, pageToken);
}
- viewStore.put(identifier.getFullName(), view);
- return new MockResponse().setResponseCode(200);
- default:
- return new MockResponse().setResponseCode(404);
+ } else {
+ response = new ListViewsResponse(Collections.emptyList(), null);
+ }
+ return mockResponse(response, 200);
+ } else {
+ return new MockResponse().setResponseCode(404);
}
}
+ private List<GetViewResponse> listViewDetails(String databaseName) {
+ return viewStore.keySet().stream()
+ .map(Identifier::fromString)
+ .filter(identifier -> identifier.getDatabaseName().equals(databaseName))
+ .map(
+ identifier -> {
+ View view = viewStore.get(identifier.getFullName());
+ ViewSchema schema =
+ new ViewSchema(
+ view.rowType().getFields(),
+ view.query(),
+ view.dialects(),
+ view.comment().orElse(null),
+ view.options());
+ return new GetViewResponse("id", identifier.getTableName(), schema);
+ })
+ .collect(Collectors.toList());
+ }
+
+ private ListViewDetailsResponse generateListViewDetailsResponse(
+ List<GetViewResponse> sortedViewDetails, int maxResults, String pageToken) {
+ List<GetViewResponse> pagedViewDetails = new ArrayList<>();
+ for (GetViewResponse getViewResponse : sortedViewDetails) {
+ if (pagedViewDetails.size() < maxResults) {
+ if (pageToken == null) {
+ pagedViewDetails.add(getViewResponse);
+ pageToken = getViewResponse.getName();
+ } else if (getViewResponse.getName().compareTo(pageToken) > 0) {
+ pagedViewDetails.add(getViewResponse);
+ }
+ } else {
+ break;
+ }
+ }
+ String nextPageToken = getNextPageTokenForViews(pagedViewDetails, maxResults);
+ return new ListViewDetailsResponse(pagedViewDetails, nextPageToken);
+ }
+
private MockResponse viewHandle(RecordedRequest request, Identifier identifier)
throws Exception {
RESTResponse response;
@@ -986,4 +1388,64 @@ public class RESTCatalogServer {
}
throw new Catalog.TableNotExistException(identifier);
}
+
+ private static int getMaxResults(RecordedRequest request) {
+ String strMaxResults = request.getRequestUrl().queryParameter(MAX_RESULTS);
+ Integer maxResults =
+ Objects.nonNull(strMaxResults) ? Integer.parseInt(strMaxResults) : null;
+ if (Objects.isNull(maxResults) || maxResults <= 0) {
+ maxResults = DEFAULT_MAX_RESULTS;
+ } else {
+ maxResults = Math.min(maxResults, DEFAULT_MAX_RESULTS);
+ }
+ return maxResults;
+ }
+
+ private String getNextPageTokenForNames(List<String> names, Integer maxResults) {
+ if (names == null
+ || names.isEmpty()
+ || Objects.isNull(maxResults)
+ || names.size() < maxResults) {
+ return null;
+ }
+ // return the last name
+ return names.get(names.size() - 1);
+ }
+
+ private String getNextPageTokenForTables(List<GetTableResponse> entities, Integer maxResults) {
+ if (entities == null
+ || entities.isEmpty()
+ || Objects.isNull(maxResults)
+ || entities.size() < maxResults) {
+ return null;
+ }
+ // return the last entity name
+ return entities.get(entities.size() - 1).getName();
+ }
+
+ private String getNextPageTokenForViews(List<GetViewResponse> entities, Integer maxResults) {
+ if (entities == null
+ || entities.isEmpty()
+ || Objects.isNull(maxResults)
+ || entities.size() < maxResults) {
+ return null;
+ }
+ // return the last entity name
+ return entities.get(entities.size() - 1).getName();
+ }
+
+ private String getNextPageTokenForPartitions(List<Partition> entities, Integer maxResults) {
+ if (entities == null
+ || entities.isEmpty()
+ || Objects.isNull(maxResults)
+ || entities.size() < maxResults) {
+ return null;
+ }
+ // return the last entity name
+ return getPartitionSortKey(entities.get(entities.size() - 1));
+ }
+
+ private String getPartitionSortKey(Partition partition) {
+ return partition.spec().toString().replace("{", "").replace("}", "");
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
index dc9bf62743..124a82154a 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
@@ -18,6 +18,7 @@
package org.apache.paimon.rest;
+import org.apache.paimon.PagedList;
import org.apache.paimon.Snapshot;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
@@ -36,6 +37,7 @@ import org.apache.paimon.rest.exceptions.NotAuthorizedException;
import org.apache.paimon.rest.responses.ConfigResponse;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.Table;
import org.apache.paimon.table.sink.BatchTableCommit;
import org.apache.paimon.table.sink.BatchTableWrite;
import org.apache.paimon.table.sink.BatchWriteBuilder;
@@ -45,6 +47,7 @@ import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.TableRead;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.view.View;
import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap;
import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
@@ -57,7 +60,9 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -65,10 +70,13 @@ import java.util.Optional;
import java.util.UUID;
import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE;
+import static org.apache.paimon.CoreOptions.METASTORE_TAG_TO_PARTITION;
import static org.apache.paimon.utils.SnapshotManagerTest.createSnapshotWithMillis;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
/** Test for REST Catalog. */
@@ -137,6 +145,326 @@ class RESTCatalogTest extends CatalogTestBase {
assertEquals(headers.get(serverDefineHeaderName), serverDefineHeaderValue);
}
+ @Test
+ public void testListTables() throws Exception {
+ super.testListTables();
+
+ String databaseName = "tables_db";
+ String[] tableNames = {"table4", "table5", "table1", "table2", "table3"};
+ String[] sortedTableNames = Arrays.stream(tableNames).sorted().toArray(String[]::new);
+ Options options = new Options(this.catalog.options());
+ try (RESTCatalog restCatalog = new RESTCatalog(CatalogContext.create(options))) {
+
+ restCatalog.createDatabase(databaseName, false);
+ List<String> restTables = restCatalog.listTables(databaseName);
+ assertThat(restTables).isEmpty();
+
+ // List tables returns a list with the names of all tables in the database
+
+ for (String tableName : tableNames) {
+ restCatalog.createTable(
+ Identifier.create(databaseName, tableName), DEFAULT_TABLE_SCHEMA, false);
+ }
+ restTables = restCatalog.listTables(databaseName);
+ assertThat(restTables).containsExactly(sortedTableNames);
+
+ // List tables throws DatabaseNotExistException when the database does not exist
+ assertThatExceptionOfType(Catalog.DatabaseNotExistException.class)
+ .isThrownBy(() -> restCatalog.listTables("non_existing_db"));
+ }
+ }
+
+ @Test
+ public void testListTablesPaged() throws Exception {
+ // List tables paged returns an empty list when there are no tables in the database
+ String databaseName = "tables_paged_db";
+ catalog.createDatabase(databaseName, false);
+ PagedList<String> pagedTables = catalog.listTablesPaged(databaseName, null, null);
+ assertThat(pagedTables.getElements()).isEmpty();
+ assertNull(pagedTables.getNextPageToken());
+
+ String[] tableNames = {"table1", "table2", "table3", "abd", "def", "opr"};
+ for (String tableName : tableNames) {
+ catalog.createTable(
+ Identifier.create(databaseName, tableName), DEFAULT_TABLE_SCHEMA, false);
+ }
+
+ // when maxResults is null or 0, the page length is set to a server configured value
+ String[] sortedTableNames = Arrays.stream(tableNames).sorted().toArray(String[]::new);
+ pagedTables = catalog.listTablesPaged(databaseName, null, null);
+ List<String> tables = pagedTables.getElements();
+ assertThat(tables).containsExactly(sortedTableNames);
+ assertNull(pagedTables.getNextPageToken());
+
+ // when maxResults is greater than 0, the page length is the minimum of this value and a
+ // server configured value
+ // when pageToken is null, will list tables from the beginning
+ int maxResults = 2;
+ pagedTables = catalog.listTablesPaged(databaseName, maxResults, null);
+ tables = pagedTables.getElements();
+ assertEquals(maxResults, tables.size());
+ assertThat(tables).containsExactly("abd", "def");
+ assertEquals("def", pagedTables.getNextPageToken());
+
+ // when pageToken is not null, will list tables from the pageToken (exclusive)
+ pagedTables =
+ catalog.listTablesPaged(databaseName, maxResults, pagedTables.getNextPageToken());
+ tables = pagedTables.getElements();
+ assertEquals(maxResults, tables.size());
+ assertThat(tables).containsExactly("opr", "table1");
+ assertEquals("table1", pagedTables.getNextPageToken());
+
+ pagedTables =
+ catalog.listTablesPaged(databaseName, maxResults, pagedTables.getNextPageToken());
+ tables = pagedTables.getElements();
+ assertEquals(maxResults, tables.size());
+ assertThat(tables).containsExactly("table2", "table3");
+ assertEquals("table3", pagedTables.getNextPageToken());
+
+ pagedTables =
+ catalog.listTablesPaged(databaseName, maxResults, pagedTables.getNextPageToken());
+ tables = pagedTables.getElements();
+ assertEquals(0, tables.size());
+ assertNull(pagedTables.getNextPageToken());
+
+ maxResults = 8;
+ pagedTables = catalog.listTablesPaged(databaseName, maxResults, null);
+ tables = pagedTables.getElements();
+ String[] expectedTableNames = Arrays.stream(tableNames).sorted().toArray(String[]::new);
+ assertThat(tables).containsExactly(expectedTableNames);
+ assertNull(pagedTables.getNextPageToken());
+
+ pagedTables = catalog.listTablesPaged(databaseName, maxResults, "table1");
+ tables = pagedTables.getElements();
+ assertEquals(2, tables.size());
+ assertThat(tables).containsExactly("table2", "table3");
+ assertNull(pagedTables.getNextPageToken());
+
+ // List tables throws DatabaseNotExistException when the database does not exist
+ assertThatExceptionOfType(Catalog.DatabaseNotExistException.class)
+ .isThrownBy(() -> catalog.listTables("non_existing_db"));
+ }
+
+ @Test
+ public void testListTableDetailsPaged() throws Exception {
+ // List table details returns an empty list when there are no tables in the database
+ String databaseName = "table_details_paged_db";
+ catalog.createDatabase(databaseName, false);
+ PagedList<Table> pagedTableDetails =
+ catalog.listTableDetailsPaged(databaseName, null, null);
+ assertThat(pagedTableDetails.getElements()).isEmpty();
+ assertNull(pagedTableDetails.getNextPageToken());
+
+ String[] tableNames = {"table1", "table2", "table3", "abd", "def", "opr"};
+ String[] expectedTableNames = Arrays.stream(tableNames).sorted().toArray(String[]::new);
+ for (String tableName : tableNames) {
+ catalog.createTable(
+ Identifier.create(databaseName, tableName), DEFAULT_TABLE_SCHEMA, false);
+ }
+
+ pagedTableDetails = catalog.listTableDetailsPaged(databaseName, null, null);
+ assertPagedTableDetails(pagedTableDetails, tableNames.length, expectedTableNames);
+ assertNull(pagedTableDetails.getNextPageToken());
+
+ int maxResults = 2;
+ pagedTableDetails = catalog.listTableDetailsPaged(databaseName, maxResults, null);
+ assertPagedTableDetails(pagedTableDetails, maxResults, "abd", "def");
+ assertEquals("def", pagedTableDetails.getNextPageToken());
+
+ pagedTableDetails =
+ catalog.listTableDetailsPaged(
+ databaseName, maxResults, pagedTableDetails.getNextPageToken());
+ assertPagedTableDetails(pagedTableDetails, maxResults, "opr", "table1");
+ assertEquals("table1", pagedTableDetails.getNextPageToken());
+
+ pagedTableDetails =
+ catalog.listTableDetailsPaged(
+ databaseName, maxResults, pagedTableDetails.getNextPageToken());
+ assertPagedTableDetails(pagedTableDetails, maxResults, "table2", "table3");
+ assertEquals("table3", pagedTableDetails.getNextPageToken());
+
+ pagedTableDetails =
+ catalog.listTableDetailsPaged(
+ databaseName, maxResults, pagedTableDetails.getNextPageToken());
+ assertEquals(0, pagedTableDetails.getElements().size());
+ assertNull(pagedTableDetails.getNextPageToken());
+
+ maxResults = 8;
+ pagedTableDetails = catalog.listTableDetailsPaged(databaseName, maxResults, null);
+ assertPagedTableDetails(
+ pagedTableDetails, Math.min(maxResults, tableNames.length), expectedTableNames);
+ assertNull(pagedTableDetails.getNextPageToken());
+
+ String pageToken = "table1";
+ pagedTableDetails = catalog.listTableDetailsPaged(databaseName, maxResults, pageToken);
+ assertPagedTableDetails(pagedTableDetails, 2, "table2", "table3");
+ assertNull(pagedTableDetails.getNextPageToken());
+
+ // List table details throws DatabaseNotExistException when the database does not exist
+ final int finalMaxResults = maxResults;
+ assertThatExceptionOfType(Catalog.DatabaseNotExistException.class)
+ .isThrownBy(
+ () ->
+ catalog.listTableDetailsPaged(
+ "non_existing_db", finalMaxResults, pageToken));
+ }
+
+ @Test
+ void testListViews() throws Exception {
+ Options options = new Options(this.catalog.options());
+ String databaseName;
+ List<String> views;
+ String[] viewNames = new String[] {"view1", "view2", "view3", "abd", "def", "opr", "xyz"};
+ String[] sortedViewNames = Arrays.stream(viewNames).sorted().toArray(String[]::new);
+ List<String> restViews;
+ try (RESTCatalog restCatalog = new RESTCatalog(CatalogContext.create(options))) {
+
+ // List tables returns an empty list when there are no tables in the database
+ databaseName = "views_paged_db";
+ restCatalog.createDatabase(databaseName, false);
+ views = restCatalog.listViews(databaseName);
+ assertThat(views).isEmpty();
+
+ View view = buildView(databaseName);
+
+ for (String viewName : viewNames) {
+ restCatalog.createView(Identifier.create(databaseName, viewName), view, false);
+ }
+
+ // when maxResults is null or 0, the page length is set to a server configured value
+ restViews = restCatalog.listViews(databaseName);
+ }
+ assertThat(restViews).containsExactly(sortedViewNames);
+ }
+
+ @Test
+ public void testListViewsPaged() throws Exception {
+ if (!supportsView()) {
+ return;
+ }
+
+ // List views returns an empty list when there are no views in the database
+ String databaseName = "views_paged_db";
+ catalog.createDatabase(databaseName, false);
+ PagedList<String> pagedViews = catalog.listViewsPaged(databaseName, null, null);
+ assertThat(pagedViews.getElements()).isEmpty();
+ assertNull(pagedViews.getNextPageToken());
+
+ // List views paged returns a list with the names of all views in the database in all
+ // catalogs except RestCatalog
+ // even if the maxResults or pageToken is not null
+ View view = buildView(databaseName);
+ String[] viewNames = {"view1", "view2", "view3", "abd", "def", "opr"};
+ String[] sortedViewNames = Arrays.stream(viewNames).sorted().toArray(String[]::new);
+ for (String viewName : viewNames) {
+ catalog.createView(Identifier.create(databaseName, viewName), view, false);
+ }
+
+ pagedViews = catalog.listViewsPaged(databaseName, null, null);
+ assertThat(pagedViews.getElements()).containsExactly(sortedViewNames);
+ assertNull(pagedViews.getNextPageToken());
+
+ int maxResults = 2;
+ pagedViews = catalog.listViewsPaged(databaseName, maxResults, null);
+ assertPagedViews(pagedViews, "abd", "def");
+ assertEquals("def", pagedViews.getNextPageToken());
+
+ pagedViews =
+ catalog.listViewsPaged(databaseName, maxResults, pagedViews.getNextPageToken());
+ assertPagedViews(pagedViews, "opr", "view1");
+ assertEquals("view1", pagedViews.getNextPageToken());
+
+ pagedViews =
+ catalog.listViewsPaged(databaseName, maxResults, pagedViews.getNextPageToken());
+ assertPagedViews(pagedViews, "view2", "view3");
+ assertEquals("view3", pagedViews.getNextPageToken());
+
+ maxResults = 8;
+ String[] expectedViewNames = Arrays.stream(viewNames).sorted().toArray(String[]::new);
+ pagedViews = catalog.listViewsPaged(databaseName, maxResults, null);
+ assertPagedViews(pagedViews, expectedViewNames);
+ assertNull(pagedViews.getNextPageToken());
+
+ String pageToken = "view1";
+ pagedViews = catalog.listViewsPaged(databaseName, maxResults, pageToken);
+ assertPagedViews(pagedViews, "view2", "view3");
+ assertNull(pagedViews.getNextPageToken());
+
+ // List views throws DatabaseNotExistException when the database does not exist
+ final int finalMaxResults = 9;
+ assertThatExceptionOfType(Catalog.DatabaseNotExistException.class)
+ .isThrownBy(
+ () ->
+ catalog.listViewsPaged(
+ "non_existing_db", finalMaxResults, pageToken));
+ }
+
+ @Test
+ public void testListViewDetailsPaged() throws Exception {
+ // List view details returns an empty list when there are no views in the database
+ String databaseName = "view_details_paged_db";
+ catalog.createDatabase(databaseName, false);
+ PagedList<View> pagedViewDetails = catalog.listViewDetailsPaged(databaseName, null, null);
+ assertThat(pagedViewDetails.getElements()).isEmpty();
+ assertNull(pagedViewDetails.getNextPageToken());
+
+ String[] viewNames = {"view1", "view2", "view3", "abd", "def", "opr"};
+ View view = buildView(databaseName);
+ for (String viewName : viewNames) {
+ catalog.createView(Identifier.create(databaseName, viewName), view, false);
+ }
+
+ pagedViewDetails = catalog.listViewDetailsPaged(databaseName, null, null);
+ assertPagedViewDetails(pagedViewDetails, view, viewNames.length, viewNames);
+ assertNull(pagedViewDetails.getNextPageToken());
+
+ int maxResults = 2;
+ pagedViewDetails = catalog.listViewDetailsPaged(databaseName, maxResults, null);
+ assertPagedViewDetails(pagedViewDetails, view, maxResults, "abd", "def");
+ assertEquals("def", pagedViewDetails.getNextPageToken());
+
+ pagedViewDetails =
+ catalog.listViewDetailsPaged(
+ databaseName, maxResults, pagedViewDetails.getNextPageToken());
+ assertPagedViewDetails(pagedViewDetails, view, maxResults, "opr", "view1");
+ assertEquals("view1", pagedViewDetails.getNextPageToken());
+
+ pagedViewDetails =
+ catalog.listViewDetailsPaged(
+ databaseName, maxResults, pagedViewDetails.getNextPageToken());
+ assertPagedViewDetails(pagedViewDetails, view, maxResults, "view2", "view3");
+ assertEquals("view3", pagedViewDetails.getNextPageToken());
+
+ pagedViewDetails =
+ catalog.listViewDetailsPaged(
+ databaseName, maxResults, pagedViewDetails.getNextPageToken());
+ assertEquals(0, pagedViewDetails.getElements().size());
+ assertNull(pagedViewDetails.getNextPageToken());
+
+ maxResults = 8;
+ pagedViewDetails = catalog.listViewDetailsPaged(databaseName, maxResults, null);
+ String[] expectedViewNames = Arrays.stream(viewNames).sorted().toArray(String[]::new);
+ assertPagedViewDetails(
+ pagedViewDetails,
+ view,
+ Math.min(maxResults, expectedViewNames.length),
+ expectedViewNames);
+ assertNull(pagedViewDetails.getNextPageToken());
+
+ String pageToken = "view1";
+ pagedViewDetails = catalog.listViewDetailsPaged(databaseName, maxResults, pageToken);
+ assertPagedViewDetails(pagedViewDetails, view, 2, "view2", "view3");
+ assertNull(pagedViewDetails.getNextPageToken());
+
+ // List view details throws DatabaseNotExistException when the database does not exist
+ final int finalMaxResults = maxResults;
+ assertThatExceptionOfType(Catalog.DatabaseNotExistException.class)
+ .isThrownBy(
+ () ->
+ catalog.listViewDetailsPaged(
+ "non_existing_db", finalMaxResults, pageToken));
+ }
+
@Test
void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception {
Identifier identifier = Identifier.create("test_db", "test_table");
@@ -156,6 +484,138 @@ class RESTCatalogTest extends CatalogTestBase {
assertEquals(0, result.size());
}
+ @Test
+ void testListPartitions() throws Exception {
+ List<Map<String, String>> partitionSpecs =
+ Arrays.asList(
+ Collections.singletonMap("dt", "20250101"),
+ Collections.singletonMap("dt", "20250102"),
+ Collections.singletonMap("dt", "20240102"),
+ Collections.singletonMap("dt", "20260101"),
+ Collections.singletonMap("dt", "20250104"),
+ Collections.singletonMap("dt", "20250103"));
+ Map[] sortedSpecs =
+ partitionSpecs.stream()
+ .sorted(Comparator.comparing(i -> i.get("dt")))
+ .toArray(Map[]::new);
+
+ Options options = new Options(this.catalog.options());
+ Identifier identifier;
+ try (RESTCatalog restCatalog = new RESTCatalog(CatalogContext.create(options))) {
+
+ String databaseName = "partitions_db";
+ identifier = Identifier.create(databaseName, "table");
+ Schema schema =
+ Schema.newBuilder()
+ .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(METASTORE_TAG_TO_PARTITION.key(), "dt")
+ .column("col", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .partitionKeys("dt")
+ .build();
+
+ restCatalog.createDatabase(databaseName, true);
+ restCatalog.createTable(identifier, schema, true);
+ restCatalog.createPartitions(identifier, partitionSpecs);
+
+ List<Partition> restPartitions = restCatalog.listPartitions(identifier);
+ assertThat(restPartitions.stream().map(Partition::spec)).containsExactly(sortedSpecs);
+
+ assertThatExceptionOfType(Catalog.TableNotExistException.class)
+ .isThrownBy(
+ () ->
+ restCatalog.listPartitions(
+ Identifier.create(databaseName, "non_existing_table")));
+ }
+ }
+
+ @Test
+ public void testListPartitionsPaged() throws Exception {
+ String databaseName = "partitions_paged_db";
+ List<Map<String, String>> partitionSpecs =
+ Arrays.asList(
+ Collections.singletonMap("dt", "20250101"),
+ Collections.singletonMap("dt", "20250102"),
+ Collections.singletonMap("dt", "20240102"),
+ Collections.singletonMap("dt", "20260101"),
+ Collections.singletonMap("dt", "20250104"),
+ Collections.singletonMap("dt", "20250103"));
+ catalog.dropDatabase(databaseName, true, true);
+ catalog.createDatabase(databaseName, true);
+ Identifier identifier = Identifier.create(databaseName, "table");
+ catalog.createTable(
+ identifier,
+ Schema.newBuilder()
+ .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(METASTORE_TAG_TO_PARTITION.key(), "dt")
+ .column("col", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .partitionKeys("dt")
+ .build(),
+ true);
+
+ catalog.createPartitions(identifier, partitionSpecs);
+ PagedList<Partition> pagedPartitions = catalog.listPartitionsPaged(identifier, null, null);
+ Map[] sortedSpecs =
+ partitionSpecs.stream()
+ .sorted(Comparator.comparing(i -> i.get("dt")))
+ .toArray(Map[]::new);
+ assertPagedPartitions(pagedPartitions, partitionSpecs.size(), sortedSpecs);
+
+ int maxResults = 2;
+ pagedPartitions = catalog.listPartitionsPaged(identifier, maxResults, null);
+ assertPagedPartitions(
+ pagedPartitions, maxResults, partitionSpecs.get(2), partitionSpecs.get(0));
+ assertEquals("dt=20250101", pagedPartitions.getNextPageToken());
+
+ pagedPartitions =
+ catalog.listPartitionsPaged(
+ identifier, maxResults, pagedPartitions.getNextPageToken());
+ assertPagedPartitions(
+ pagedPartitions, maxResults, partitionSpecs.get(1), partitionSpecs.get(5));
+ assertEquals("dt=20250103", pagedPartitions.getNextPageToken());
+
+ pagedPartitions =
+ catalog.listPartitionsPaged(
+ identifier, maxResults, pagedPartitions.getNextPageToken());
+ assertPagedPartitions(
+ pagedPartitions, maxResults, partitionSpecs.get(4), partitionSpecs.get(3));
+ assertEquals("dt=20260101", pagedPartitions.getNextPageToken());
+
+ pagedPartitions =
+ catalog.listPartitionsPaged(
+ identifier, maxResults, pagedPartitions.getNextPageToken());
+ assertThat(pagedPartitions.getElements()).isEmpty();
+ assertNull(pagedPartitions.getNextPageToken());
+
+ maxResults = 8;
+ pagedPartitions = catalog.listPartitionsPaged(identifier, maxResults, null);
+
+ assertPagedPartitions(
+ pagedPartitions, Math.min(maxResults, partitionSpecs.size()), sortedSpecs);
+ assertNull(pagedPartitions.getNextPageToken());
+
+ pagedPartitions = catalog.listPartitionsPaged(identifier, maxResults, "dt=20250101");
+ assertPagedPartitions(
+ pagedPartitions,
+ 4,
+ partitionSpecs.get(1),
+ partitionSpecs.get(5),
+ partitionSpecs.get(4),
+ partitionSpecs.get(3));
+ assertNull(pagedPartitions.getNextPageToken());
+
+ // List partitions paged throws TableNotExistException when the table does not exist
+ final int finalMaxResults = maxResults;
+ assertThatExceptionOfType(Catalog.TableNotExistException.class)
+ .isThrownBy(
+ () ->
+ catalog.listPartitionsPaged(
+ Identifier.create(databaseName, "non_existing_table"),
+ finalMaxResults,
+ "dt=20250101"));
+ }
+
@Test
void testRefreshFileIO() throws Exception {
this.catalog = initDataTokenCatalog();
@@ -308,6 +768,11 @@ class RESTCatalogTest extends CatalogTestBase {
return true;
}
+ @Override
+ protected boolean supportPagedList() {
+ return true;
+ }
+
@Override
protected boolean supportsAlterDatabase() {
return true;
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index d102ec0c53..afb29338d2 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -19,6 +19,7 @@
package org.apache.paimon.hive;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.PagedList;
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.catalog.AbstractCatalog;
import org.apache.paimon.catalog.Catalog;
@@ -90,6 +91,7 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -851,6 +853,35 @@ public class HiveCatalog extends AbstractCatalog {
}
}
+ @Override
+ public PagedList<View> listViewDetailsPaged(
+ String databaseName, Integer maxResults, String pageToken)
+ throws DatabaseNotExistException {
+ if (isSystemDatabase(databaseName)) {
+ return new PagedList<>(Collections.emptyList(), null);
+ }
+ getDatabase(databaseName);
+
+ PagedList<String> pagedViewNames = listViewsPaged(databaseName, maxResults, pageToken);
+ return new PagedList<>(
+ pagedViewNames.getElements().stream()
+ .map(
+ viewName -> {
+ try {
+ return getView(Identifier.create(databaseName, viewName));
+ } catch (ViewNotExistException ignored) {
+ LOG.warn(
+ "view {}.{} does not exist",
+ databaseName,
+ viewName);
+ return null;
+ }
+ })
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList()),
+ pagedViewNames.getNextPageToken());
+ }
+
@Override
public void renameView(Identifier fromView, Identifier toView, boolean ignoreIfNotExists)
throws ViewNotExistException, ViewAlreadyExistException {
diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml
index 42b302ca7f..982f91bf85 100644
--- a/paimon-open-api/rest-catalog-open-api.yaml
+++ b/paimon-open-api/rest-catalog-open-api.yaml
@@ -215,6 +215,17 @@ paths:
required: true
schema:
type: string
+ - name: maxResults
+ in: path
+ required: false
+ schema:
+ type: integer
+ format: int32
+ - name: pageToken
+ in: path
+ required: false
+ schema:
+ type: string
responses:
"200":
description: OK
@@ -250,6 +261,43 @@ paths:
description: Success, no content
"500":
description: Internal Server Error
+ /v1/{prefix}/databases/{database}/table-details:
+ get:
+ tags:
+ - table
+ summary: List table details
+ operationId: listTableDetails
+ parameters:
+ - name: prefix
+ in: path
+ required: true
+ schema:
+ type: string
+ - name: database
+ in: path
+ required: true
+ schema:
+ type: string
+ - name: maxResults
+ in: path
+ required: false
+ schema:
+ type: integer
+ format: int32
+ - name: pageToken
+ in: path
+ required: false
+ schema:
+ type: string
+ responses:
+ "200":
+ description: OK
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ListTableDetailsResponse'
+ "500":
+ description: Internal Server Error
/v1/{prefix}/databases/{database}/tables/{table}:
get:
tags:
@@ -540,6 +588,17 @@ paths:
required: true
schema:
type: string
+ - name: maxResults
+ in: path
+ required: false
+ schema:
+ type: integer
+ format: int32
+ - name: pageToken
+ in: path
+ required: false
+ schema:
+ type: string
responses:
"200":
description: OK
@@ -879,6 +938,17 @@ paths:
required: true
schema:
type: string
+ - name: maxResults
+ in: path
+ required: false
+ schema:
+ type: integer
+ format: int32
+ - name: pageToken
+ in: path
+ required: false
+ schema:
+ type: string
responses:
"200":
description: OK
@@ -920,6 +990,43 @@ paths:
$ref: '#/components/schemas/ErrorResponse'
"500":
description: Internal Server Error
+ /v1/{prefix}/databases/{database}/view-details:
+ get:
+ tags:
+ - view
+ summary: List view details
+ operationId: listViewDetails
+ parameters:
+ - name: prefix
+ in: path
+ required: true
+ schema:
+ type: string
+ - name: database
+ in: path
+ required: true
+ schema:
+ type: string
+ - name: maxResults
+ in: path
+ required: false
+ schema:
+ type: integer
+ format: int32
+ - name: pageToken
+ in: path
+ required: false
+ schema:
+ type: string
+ responses:
+ "200":
+ description: OK
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ListViewDetailsResponse'
+ "500":
+ description: Internal Server Error
/v1/{prefix}/databases/{database}/views/{view}:
get:
tags:
@@ -1487,6 +1594,17 @@ components:
type: array
items:
type: string
+ nextPageToken:
+ type: string
+ ListTableDetailsResponse:
+ type: object
+ properties:
+ tableDetails:
+ type: array
+ items:
+ $ref: '#/components/schemas/GetTableResponse'
+ nextPageToken:
+ type: string
ConfigResponse:
type: object
properties:
@@ -1505,6 +1623,8 @@ components:
type: array
items:
$ref: '#/components/schemas/Partition'
+ nextPageToken:
+ type: string
CreateBranchRequest:
type: object
properties:
@@ -1541,6 +1661,17 @@ components:
type: array
items:
type: string
+ nextPageToken:
+ type: string
+ ListViewDetailsResponse:
+ type: object
+ properties:
+ viewDetails:
+ type: array
+ items:
+ $ref: '#/components/schemas/GetViewResponse'
+ nextPageToken:
+ type: string
ViewSchema:
type: object
properties:
diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java
index 8dbff3b8c2..a3b7e3f422 100644
--- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java
+++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java
@@ -45,7 +45,9 @@ import org.apache.paimon.rest.responses.GetViewResponse;
import org.apache.paimon.rest.responses.ListBranchesResponse;
import org.apache.paimon.rest.responses.ListDatabasesResponse;
import org.apache.paimon.rest.responses.ListPartitionsResponse;
+import org.apache.paimon.rest.responses.ListTableDetailsResponse;
import org.apache.paimon.rest.responses.ListTablesResponse;
+import org.apache.paimon.rest.responses.ListViewDetailsResponse;
import org.apache.paimon.rest.responses.ListViewsResponse;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.IntType;
@@ -226,8 +228,47 @@ public class RESTCatalogController {
})
@GetMapping("/v1/{prefix}/databases/{database}/tables")
public ListTablesResponse listTables(
- @PathVariable String prefix, @PathVariable String database) {
- return new ListTablesResponse(ImmutableList.of("user"));
+ @PathVariable String prefix,
+ @PathVariable String database,
+ @PathVariable Integer maxResults,
+ @PathVariable String pageToken) {
+ // paged list tables in this database with provided maxResults and pageToken
+ return new ListTablesResponse(ImmutableList.of("user"), null);
+ }
+
+ @Operation(
+ summary = "List table details",
+ tags = {"table"})
+ @ApiResponses({
+ @ApiResponse(
+ responseCode = "200",
+ content = {
+ @Content(schema = @Schema(implementation = ListTableDetailsResponse.class))
+ }),
+ @ApiResponse(
+ responseCode = "500",
+ content = {@Content(schema = @Schema())})
+ })
+ @GetMapping("/v1/{prefix}/databases/{database}/table-details")
+ public ListTableDetailsResponse listTableDetails(
+ @PathVariable String prefix,
+ @PathVariable String database,
+ @PathVariable Integer maxResults,
+ @PathVariable String pageToken) {
+ // paged list table details in this database with provided maxResults and pageToken
+ GetTableResponse singleTable =
+ new GetTableResponse(
+ UUID.randomUUID().toString(),
+ "",
+ false,
+ 1,
+ new org.apache.paimon.schema.Schema(
+ ImmutableList.of(),
+ ImmutableList.of(),
+ ImmutableList.of(),
+ new HashMap<>(),
+ "test-comment"));
+ return new ListTableDetailsResponse(ImmutableList.of(singleTable), null);
}
@Operation(
@@ -435,7 +476,10 @@ public class RESTCatalogController {
public ListPartitionsResponse listPartitions(
@PathVariable String prefix,
@PathVariable String database,
- @PathVariable String table) {
+ @PathVariable String table,
+ @PathVariable Integer maxResults,
+ @PathVariable String pageToken) {
+ // paged list partitions in this table with provided maxResults and pageToken
Map<String, String> spec = new HashMap<>();
spec.put("f1", "1");
Partition partition = new Partition(spec, 1, 2, 3, 4);
@@ -615,8 +659,50 @@ public class RESTCatalogController {
content = {@Content(schema = @Schema())})
})
@GetMapping("/v1/{prefix}/databases/{database}/views")
- public ListViewsResponse listViews(@PathVariable String prefix, @PathVariable String database) {
- return new ListViewsResponse(ImmutableList.of("view1"));
+ public ListViewsResponse listViews(
+ @PathVariable String prefix,
+ @PathVariable String database,
+ @PathVariable Integer maxResults,
+ @PathVariable String pageToken) {
+ // support paged list views in this database with provided maxResults and pageToken
+ return new ListViewsResponse(ImmutableList.of("view1"), null);
+ }
+
+ @Operation(
+ summary = "List view details",
+ tags = {"view"})
+ @ApiResponses({
+ @ApiResponse(
+ responseCode = "200",
+ content = {@Content(schema = @Schema(implementation = ListViewsResponse.class))}),
+ @ApiResponse(
+ responseCode = "404",
+ description = "Resource not found",
+ content = {@Content(schema = @Schema(implementation = ErrorResponse.class))}),
+ @ApiResponse(
+ responseCode = "500",
+ content = {@Content(schema = @Schema())})
+ })
+ @GetMapping("/v1/{prefix}/databases/{database}/views")
+ public ListViewDetailsResponse listViewDetails(
+ @PathVariable String prefix,
+ @PathVariable String database,
+ @PathVariable Integer maxResults,
+ @PathVariable String pageToken) {
+ // paged list view details in this database with provided maxResults and pageToken
+ List<DataField> fields =
+ Arrays.asList(
+ new DataField(0, "f0", new IntType()),
+ new DataField(1, "f1", new IntType()));
+ ViewSchema schema =
+ new ViewSchema(
+ fields,
+ "select * from t1",
+ Collections.emptyMap(),
+ "comment",
+ Collections.singletonMap("pt", "1"));
+ GetViewResponse singleView = new GetViewResponse("id", "name", schema);
+ return new ListViewDetailsResponse(ImmutableList.of(singleView), null);
}
@Operation(