You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by yu...@apache.org on 2024/09/17 00:03:26 UTC
(polaris) branch main updated: Add a config option to limit request body size (#276)
This is an automated email from the ASF dual-hosted git repository.
yufei pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 68b248aa Add a config option to limit request body size (#276)
68b248aa is described below
commit 68b248aa98a527fe2f15a8920d7524adf288d6ed
Author: Andrew Guterman <an...@gmail.com>
AuthorDate: Mon Sep 16 17:03:22 2024 -0700
Add a config option to limit request body size (#276)
---
polaris-server.yml | 3 ++
.../apache/polaris/service/PolarisApplication.java | 14 +++++
.../service/config/PolarisApplicationConfig.java | 20 +++++++
.../throttling/RequestThrottlingErrorResponse.java | 45 ++++++++++++++++
.../StreamReadConstraintsExceptionMapper.java | 42 +++++++++++++++
.../service/PolarisApplicationIntegrationTest.java | 62 ++++++++++++++++++++++
.../resources/polaris-server-integrationtest.yml | 3 ++
7 files changed, 189 insertions(+)
diff --git a/polaris-server.yml b/polaris-server.yml
index 6bf8dcc6..3c751d68 100644
--- a/polaris-server.yml
+++ b/polaris-server.yml
@@ -163,3 +163,6 @@ logging:
archivedLogFilenamePattern: ./logs/polaris-%d.log.gz
# The maximum number of log files to archive.
archivedFileCount: 14
+
+# Limits the size of request bodies sent to Polaris. -1 means no limit.
+maxRequestBodyBytes: -1
diff --git a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
index 3c73d0a5..5706fb27 100644
--- a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
+++ b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java
@@ -20,9 +20,11 @@ package org.apache.polaris.service;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
+import static org.apache.polaris.service.config.PolarisApplicationConfig.REQUEST_BODY_BYTES_NO_LIMIT;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.core.StreamReadConstraints;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
@@ -104,6 +106,7 @@ import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler;
import org.apache.polaris.service.task.TableCleanupTaskHandler;
import org.apache.polaris.service.task.TaskExecutorImpl;
import org.apache.polaris.service.task.TaskFileIOSupplier;
+import org.apache.polaris.service.throttling.StreamReadConstraintsExceptionMapper;
import org.apache.polaris.service.tracing.OpenTelemetryAware;
import org.apache.polaris.service.tracing.TracingFilter;
import org.eclipse.jetty.servlets.CrossOriginFilter;
@@ -273,6 +276,17 @@ public class PolarisApplication extends Application<PolarisApplicationConfig> {
objectMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
objectMapper.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy());
+
+ long maxRequestBodyBytes = configuration.getMaxRequestBodyBytes();
+ if (maxRequestBodyBytes != REQUEST_BODY_BYTES_NO_LIMIT) {
+ objectMapper
+ .getFactory()
+ .setStreamReadConstraints(
+ StreamReadConstraints.builder().maxDocumentLength(maxRequestBodyBytes).build());
+ LOGGER.info("Limiting request body size to {} bytes", maxRequestBodyBytes);
+ }
+
+ environment.jersey().register(new StreamReadConstraintsExceptionMapper());
RESTSerializers.registerAll(objectMapper);
Serializers.registerSerializers(objectMapper);
environment.jersey().register(new IcebergJsonProcessingExceptionMapper());
diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java b/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java
index f8b7371f..56b83b97 100644
--- a/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java
+++ b/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java
@@ -19,6 +19,7 @@
package org.apache.polaris.service.config;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
import io.dropwizard.core.Configuration;
import java.util.HashMap;
import java.util.List;
@@ -55,6 +56,9 @@ public class PolarisApplicationConfig extends Configuration {
private String awsSecretKey;
private FileIOFactory fileIOFactory;
+ public static final long REQUEST_BODY_BYTES_NO_LIMIT = -1;
+ private long maxRequestBodyBytes = REQUEST_BODY_BYTES_NO_LIMIT;
+
@JsonProperty("metaStoreManager")
public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) {
this.metaStoreManagerFactory = metaStoreManagerFactory;
@@ -146,6 +150,22 @@ public class PolarisApplicationConfig extends Configuration {
this.configurationStore = new DefaultConfigurationStore(featureConfiguration);
}
+ @JsonProperty("maxRequestBodyBytes")
+ public void setMaxRequestBodyBytes(long maxRequestBodyBytes) {
+ // The underlying library that we use to implement the limit treats all values <= 0 as the
+ // same, so we block all but -1 to prevent ambiguity.
+ Preconditions.checkArgument(
+ maxRequestBodyBytes == -1 || maxRequestBodyBytes > 0,
+ "maxRequestBodyBytes must be a positive integer or %s to specify no limit.",
+ REQUEST_BODY_BYTES_NO_LIMIT);
+
+ this.maxRequestBodyBytes = maxRequestBodyBytes;
+ }
+
+ public long getMaxRequestBodyBytes() {
+ return maxRequestBodyBytes;
+ }
+
public PolarisConfigurationStore getConfigurationStore() {
return configurationStore;
}
diff --git a/polaris-service/src/main/java/org/apache/polaris/service/throttling/RequestThrottlingErrorResponse.java b/polaris-service/src/main/java/org/apache/polaris/service/throttling/RequestThrottlingErrorResponse.java
new file mode 100644
index 00000000..8b1b560a
--- /dev/null
+++ b/polaris-service/src/main/java/org/apache/polaris/service/throttling/RequestThrottlingErrorResponse.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.throttling;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Response object for errors caused by DoS-prevention throttling mechanisms, such as request size
+ * limits
+ */
+public class RequestThrottlingErrorResponse {
+ public enum RequestThrottlingErrorType {
+ REQUEST_TOO_LARGE,
+ ;
+ }
+
+ private final RequestThrottlingErrorType errorType;
+
+ @JsonCreator
+ public RequestThrottlingErrorResponse(
+ @JsonProperty("error_type") RequestThrottlingErrorType errorType) {
+ this.errorType = errorType;
+ }
+
+ public RequestThrottlingErrorType getErrorType() {
+ return errorType;
+ }
+}
diff --git a/polaris-service/src/main/java/org/apache/polaris/service/throttling/StreamReadConstraintsExceptionMapper.java b/polaris-service/src/main/java/org/apache/polaris/service/throttling/StreamReadConstraintsExceptionMapper.java
new file mode 100644
index 00000000..b01dca0e
--- /dev/null
+++ b/polaris-service/src/main/java/org/apache/polaris/service/throttling/StreamReadConstraintsExceptionMapper.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.throttling;
+
+import com.fasterxml.jackson.core.exc.StreamConstraintsException;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.ext.ExceptionMapper;
+
+/**
+ * Handles exceptions during the request that are a result of stream constraints such as the request
+ * being too large
+ */
+public class StreamReadConstraintsExceptionMapper
+ implements ExceptionMapper<StreamConstraintsException> {
+
+ @Override
+ public Response toResponse(StreamConstraintsException exception) {
+ return Response.status(Response.Status.BAD_REQUEST)
+ .type(MediaType.APPLICATION_JSON_TYPE)
+ .entity(
+ new RequestThrottlingErrorResponse(
+ RequestThrottlingErrorResponse.RequestThrottlingErrorType.REQUEST_TOO_LARGE))
+ .build();
+ }
+}
diff --git a/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java
index 2cf83f1f..f116e0e8 100644
--- a/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java
+++ b/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationIntegrationTest.java
@@ -26,7 +26,9 @@ import io.dropwizard.testing.ConfigOverride;
import io.dropwizard.testing.ResourceHelpers;
import io.dropwizard.testing.junit5.DropwizardAppExtension;
import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
+import jakarta.ws.rs.ProcessingException;
import jakarta.ws.rs.client.Entity;
+import jakarta.ws.rs.client.Invocation;
import jakarta.ws.rs.core.Response;
import java.io.IOException;
import java.nio.file.Files;
@@ -74,6 +76,7 @@ import org.apache.polaris.service.config.PolarisApplicationConfig;
import org.apache.polaris.service.test.PolarisConnectionExtension;
import org.apache.polaris.service.test.PolarisRealm;
import org.apache.polaris.service.test.SnowmanCredentialsExtension;
+import org.apache.polaris.service.throttling.RequestThrottlingErrorResponse;
import org.assertj.core.api.Assertions;
import org.assertj.core.api.InstanceOfAssertFactories;
import org.junit.jupiter.api.AfterAll;
@@ -638,4 +641,63 @@ public class PolarisApplicationIntegrationTest {
.hasMessage("Malformed request: Please specify a warehouse");
}
}
+
+ @Test
+ public void testRequestHeaderTooLarge() {
+ Invocation.Builder request =
+ EXT.client()
+ .target(
+ String.format(
+ "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort()))
+ .request("application/json");
+
+ // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 definitely
+ // exceeds the limit
+ for (int i = 0; i < 1500; i++) {
+ request = request.header("header" + i, "" + i);
+ }
+
+ try {
+ try (Response response =
+ request
+ .header("Authorization", "Bearer " + userToken)
+ .header(REALM_PROPERTY_KEY, realm)
+ .post(Entity.json(new PrincipalRole("r")))) {
+ assertThat(response)
+ .returns(
+ Response.Status.REQUEST_HEADER_FIELDS_TOO_LARGE.getStatusCode(),
+ Response::getStatus);
+ }
+ } catch (ProcessingException e) {
+ // In some runtime environments the request above will return a 431 but in others it'll result
+ // in a ProcessingException from the socket being closed. The test asserts that one of those
+ // things happens.
+ }
+ }
+
+ @Test
+ public void testRequestBodyTooLarge() {
+ // The size is set to be higher than the limit in polaris-server-integrationtest.yml
+ Entity<PrincipalRole> largeRequest = Entity.json(new PrincipalRole("r".repeat(1000001)));
+
+ try (Response response =
+ EXT.client()
+ .target(
+ String.format(
+ "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort()))
+ .request("application/json")
+ .header("Authorization", "Bearer " + userToken)
+ .header(REALM_PROPERTY_KEY, realm)
+ .post(largeRequest)) {
+ assertThat(response)
+ .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus)
+ .matches(
+ r ->
+ r.readEntity(RequestThrottlingErrorResponse.class)
+ .getErrorType()
+ .equals(
+ RequestThrottlingErrorResponse.RequestThrottlingErrorType
+ .REQUEST_TOO_LARGE));
+ }
+ }
}
diff --git a/polaris-service/src/test/resources/polaris-server-integrationtest.yml b/polaris-service/src/test/resources/polaris-server-integrationtest.yml
index 2ecccf1d..d6a1aac8 100644
--- a/polaris-service/src/test/resources/polaris-server-integrationtest.yml
+++ b/polaris-service/src/test/resources/polaris-server-integrationtest.yml
@@ -149,3 +149,6 @@ logging:
archivedLogFilenamePattern: ./logs/iceberg-rest-%d.log.gz
# The maximum number of log files to archive.
archivedFileCount: 14
+
+# Limits the size of request bodies sent to Polaris. -1 means no limit.
+maxRequestBodyBytes: 1000000