You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2026/09/01 03:23:55 UTC
(impala) 04/12: IMPALA-15217: Add guards for avro.schema.url
This is an automated email from the ASF dual-hosted git repository.
MikaelSmith pushed a commit to branch branch-4.5.2
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ee0160cd182fd5101c7705330388229f23ae2726
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Wed Jun 24 15:14:55 2026 -0700
IMPALA-15217: Add guards for avro.schema.url
Adds CLI flags `avro_schema_url_allowed_schemes`,
`avro_schema_url_remote_http_enabled`, and
`avro_schema_url_http_allowed_hosts` to provide additional guards on
sending HTTP requests to unexpected hosts or reading from remote
filesystems.
Also protects against HTTP redirects and hung connections.
Adds support for HTTPS Avro schema URLs.
Assisted-by: Claude Sonnet 4.6 (VSCode)
Change-Id: I72c331ea53170644c4b2234b61dbdd880ea23e55
Reviewed-on: http://gerrit.cloudera.org:8080/24770
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Michael Smith <mi...@cloudera.com>
---
be/src/util/backend-gflag-util.cc | 20 +++
common/thrift/BackendGflags.thrift | 6 +
.../org/apache/impala/service/BackendConfig.java | 12 ++
.../org/apache/impala/util/AvroSchemaUtils.java | 59 ++++++++-
tests/authorization/test_avro_schema_url.py | 40 +++---
tests/custom_cluster/test_avro_schema_url_flags.py | 138 +++++++++++++++++++++
6 files changed, 251 insertions(+), 24 deletions(-)
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
index 292a050db..20fc6d2ab 100644
--- a/be/src/util/backend-gflag-util.cc
+++ b/be/src/util/backend-gflag-util.cc
@@ -281,6 +281,22 @@ DEFINE_int64_hidden(data_stream_sender_buffer_size_used_by_planner, -1,
"With default -1 the planner uses the old logic that is different"
"than how the backend actually works (see IMPALA-12594)");
+DEFINE_string(avro_schema_url_allowed_schemes,
+ "hdfs,s3a,abfs,abfss,gs,o3fs,ofs",
+ "Comma-separated list of URI schemes permitted for avro.schema.url when loading "
+ "schemas from a remote location. HTTP is never allowed via this setting. A URI with "
+ "no scheme is resolved against the default filesystem.");
+
+DEFINE_bool(avro_schema_url_remote_http_enabled, false,
+ "Whether to allow avro.schema.url values that use http. When enabled, the host must "
+ "also appear in avro_schema_url_http_allowed_hosts. Disabled by default to prevent "
+ "server-side request forgery.");
+
+DEFINE_string(avro_schema_url_http_allowed_hosts, "",
+ "Comma-separated list of hosts permitted for avro.schema.url when "
+ "avro_schema_url_remote_http_enabled is true. HTTP schema fetch is rejected when "
+ "this list is empty.");
+
using strings::Substitute;
namespace impala {
@@ -508,6 +524,10 @@ Status PopulateThriftBackendGflags(TBackendGflags& cfg) {
cfg.__set_is_release_build(false);
#endif
cfg.__set_enable_catalogd_ha(FLAGS_enable_catalogd_ha);
+ cfg.__set_avro_schema_url_allowed_schemes(FLAGS_avro_schema_url_allowed_schemes);
+ cfg.__set_avro_schema_url_remote_http_enabled(
+ FLAGS_avro_schema_url_remote_http_enabled);
+ cfg.__set_avro_schema_url_http_allowed_hosts(FLAGS_avro_schema_url_http_allowed_hosts);
return Status::OK();
}
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
index cfa7111ef..7f5b67377 100644
--- a/common/thrift/BackendGflags.thrift
+++ b/common/thrift/BackendGflags.thrift
@@ -316,4 +316,10 @@ struct TBackendGflags {
142: required bool enable_reading_puffin_stats
143: required string injected_group_members_debug_only
+
+ 174: required string avro_schema_url_allowed_schemes
+
+ 175: required bool avro_schema_url_remote_http_enabled
+
+ 176: required string avro_schema_url_http_allowed_hosts
}
diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
index 8cb113191..0d42032b5 100644
--- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
+++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
@@ -549,4 +549,16 @@ public class BackendConfig {
public String getInjectedGroupMembersDebugOnly() {
return backendCfg_.injected_group_members_debug_only;
}
+
+ public String getAvroSchemaUrlAllowedSchemes() {
+ return backendCfg_.avro_schema_url_allowed_schemes;
+ }
+
+ public boolean isAvroSchemaUrlRemoteHttpEnabled() {
+ return backendCfg_.avro_schema_url_remote_http_enabled;
+ }
+
+ public String getAvroSchemaUrlHttpAllowedHosts() {
+ return backendCfg_.avro_schema_url_http_allowed_hosts;
+ }
}
diff --git a/fe/src/main/java/org/apache/impala/util/AvroSchemaUtils.java b/fe/src/main/java/org/apache/impala/util/AvroSchemaUtils.java
index 9032a46c2..eefc1ad24 100644
--- a/fe/src/main/java/org/apache/impala/util/AvroSchemaUtils.java
+++ b/fe/src/main/java/org/apache/impala/util/AvroSchemaUtils.java
@@ -19,8 +19,11 @@ package org.apache.impala.util;
import java.io.IOException;
import java.io.InputStream;
+import java.net.HttpURLConnection;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -122,7 +125,7 @@ public abstract class AvroSchemaUtils {
/**
* Gets an Avro table's JSON schema from the list of given table property search
* locations. The schema may be specified as a string literal or provided as a
- * Hadoop FileSystem or http URL that points to the schema. Apart from ensuring
+ * Hadoop FileSystem or http(s) URL that points to the schema. Apart from ensuring
* that the JSON schema is not SCHEMA_NONE, this function does not perform any
* additional validation on the returned string (e.g., it may not be a valid
* schema). Returns the Avro schema or null if none was specified in the search
@@ -155,13 +158,57 @@ public abstract class AvroSchemaUtils {
return null;
}
try {
- // TODO: Add support for https:// here.
- if (url.toLowerCase().startsWith("http://")) {
- try (InputStream in = new URL(url).openStream()) {
- return IOUtils.toString(in);
+ Path path = new Path(url);
+ String scheme = path.toUri().getScheme();
+ if (scheme != null && Arrays.asList("http", "https").contains(scheme)) {
+ if (!BackendConfig.INSTANCE.isAvroSchemaUrlRemoteHttpEnabled()) {
+ throw new AnalysisException("avro.schema.url does not permit HTTP(S). Set " +
+ "--avro_schema_url_remote_http_enabled=true to enable.");
+ }
+ String allowedHosts = BackendConfig.INSTANCE.getAvroSchemaUrlHttpAllowedHosts();
+ if (allowedHosts.isEmpty()) {
+ throw new AnalysisException("avro.schema.url HTTP(S) fetching requires " +
+ "--avro_schema_url_http_allowed_hosts to be set.");
+ }
+ String host = new URL(url).getHost();
+ if (host == null || host.isEmpty()) {
+ throw new AnalysisException("avro.schema.url HTTP(S) fetching requires a " +
+ "valid host in the URL.");
+ }
+ List<String> allowed = Arrays.asList(allowedHosts.toLowerCase().split(","));
+ if (!allowed.contains(host.toLowerCase())) {
+ throw new AnalysisException(String.format(
+ "Host '%s' is not permitted for avro.schema.url HTTP(S) fetching. " +
+ "Allowed hosts: %s", host, allowedHosts));
+ }
+ // HTTPS returns a HttpsURLConnection, which extends HttpURLConnection.
+ HttpURLConnection conn =
+ (HttpURLConnection) new URL(url).openConnection();
+ // Disable redirect following to prevent SSRF: a redirect from an
+ // allowlisted host to an internal host would otherwise bypass the
+ // allowlist check above.
+ conn.setInstanceFollowRedirects(false);
+ conn.setConnectTimeout(30_000);
+ conn.setReadTimeout(120_000);
+ int responseCode = conn.getResponseCode();
+ if (responseCode >= 300 && responseCode < 400) {
+ throw new AnalysisException(String.format(
+ "avro.schema.url HTTP(S) redirect is not permitted (host '%s' returned " +
+ "status %d). Redirects are disabled to prevent server-side request " +
+ "forgery.", host, responseCode));
+ }
+ try (InputStream in = conn.getInputStream()) {
+ return IOUtils.toString(in, StandardCharsets.UTF_8);
}
} else {
- return FileSystemUtil.readFile(new Path(url));
+ String allowedSchemes = BackendConfig.INSTANCE.getAvroSchemaUrlAllowedSchemes();
+ List<String> allowed = Arrays.asList(allowedSchemes.toLowerCase().split(","));
+ if (scheme != null && !allowed.contains(scheme.toLowerCase())) {
+ throw new AnalysisException(String.format(
+ "URI scheme '%s' is not permitted for avro.schema.url. " +
+ "Allowed schemes: %s", scheme, allowedSchemes));
+ }
+ return FileSystemUtil.readFile(path);
}
} catch (IOException ioe) {
throw new AnalysisException("Unable to read schema from given path: " + url, ioe);
diff --git a/tests/authorization/test_avro_schema_url.py b/tests/authorization/test_avro_schema_url.py
index 1dfb0059d..b9e7c0d0c 100644
--- a/tests/authorization/test_avro_schema_url.py
+++ b/tests/authorization/test_avro_schema_url.py
@@ -230,45 +230,49 @@ class TestAvroSchemaUrlAuthz(CustomClusterTestSuite):
def test_http_url_create_table_denied_without_uri_privilege(self, unique_name):
"""CREATE TABLE with an HTTP avro.schema.url fails when the user has no URI ALL."""
db = unique_name + "_db"
- http_url = "http://example.com/schema.json"
+ http_urls = ["http://example.com/schema.json", "https://example.com/schema.json"]
admin_client = self.create_impala_client()
try:
self._setup_db(admin_client, db)
- result = self.execute_query_expect_failure(
- self.client,
- "create table {db}.t stored as avro tblproperties "
- "('avro.schema.url'='{url}')".format(db=db, url=http_url),
- user=OWNER_USER)
- assert AUTHZ_ERROR in str(result)
+ for http_url in http_urls:
+ result = self.execute_query_expect_failure(
+ self.client,
+ "create table {db}.t stored as avro tblproperties "
+ "('avro.schema.url'='{url}')".format(db=db, url=http_url),
+ user=OWNER_USER)
+ assert AUTHZ_ERROR in str(result)
finally:
self._teardown_db(admin_client, db)
def test_http_url_create_table_allowed_with_uri_privilege(self, unique_name):
"""CREATE TABLE with an HTTP avro.schema.url succeeds when the user holds URI ALL."""
db = unique_name + "_db"
- http_url = "http://example.com/schema.json"
+ http_urls = ["http://example.com/schema.json", "https://example.com/schema.json"]
admin_client = self.create_impala_client()
try:
self._setup_db(admin_client, db)
- admin_client.execute("grant all on uri '{url}' to user {user}"
- .format(url=http_url, user=OWNER_USER), user=ADMIN)
+ for http_url in http_urls:
+ admin_client.execute("grant all on uri '{url}' to user {user}"
+ .format(url=http_url, user=OWNER_USER), user=ADMIN)
admin_client.execute("refresh authorization", user=ADMIN)
# The URI privilege check passes during analysis; the HTTP fetch itself may
# fail at execution time, but that is outside the scope of this test.
# We only verify that no AuthorizationException is raised.
try:
- self.execute_query_expect_success(
- self.client,
- "create table {db}.t stored as avro tblproperties "
- "('avro.schema.url'='{url}')".format(db=db, url=http_url),
- query_options={'sync_ddl': 1},
- user=OWNER_USER)
+ for http_url in http_urls:
+ self.execute_query_expect_success(
+ self.client,
+ "create table {db}.t stored as avro tblproperties "
+ "('avro.schema.url'='{url}')".format(db=db, url=http_url),
+ query_options={'sync_ddl': 1},
+ user=OWNER_USER)
except Exception as e:
assert AUTHZ_ERROR not in str(e), \
"Got unexpected AuthorizationException: {0}".format(e)
finally:
- admin_client.execute("revoke all on uri '{url}' from user {user}"
- .format(url=http_url, user=OWNER_USER), user=ADMIN)
+ for http_url in http_urls:
+ admin_client.execute("revoke all on uri '{url}' from user {user}"
+ .format(url=http_url, user=OWNER_USER), user=ADMIN)
self._teardown_db(admin_client, db)
# ---------------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_avro_schema_url_flags.py b/tests/custom_cluster/test_avro_schema_url_flags.py
new file mode 100644
index 000000000..955c3898a
--- /dev/null
+++ b/tests/custom_cluster/test_avro_schema_url_flags.py
@@ -0,0 +1,138 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Authorization tests for the avro.schema.url table property.
+#
+# Both HDFS and HTTP values of avro.schema.url are now subject to Ranger URI
+# privilege enforcement via HdfsUri.analyze(). A user must hold the ALL
+# privilege on the URI to use it in CREATE TABLE or ALTER TABLE statements.
+# When avro.schema.literal is also present, the URL is never evaluated and
+# no URI privilege is required.
+
+import pytest
+
+from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.util.filesystem_utils import FILESYSTEM_URI_SCHEME
+
+# ---------------------------------------------------------------------------
+# Error substrings for flag-enforcement checks
+# ---------------------------------------------------------------------------
+HTTP_DISABLED_ERROR = "does not permit HTTP(S)"
+HOST_NOT_ALLOWED_ERROR = "is not permitted for avro.schema.url HTTP(S) fetching"
+SCHEME_NOT_ALLOWED_ERROR = "is not permitted for avro.schema.url"
+
+# The impalad debug web UI returns JSON at this URL - used as a convenient
+# HTTP test server that is always reachable within the cluster.
+VARZ_HTTP_URL = "http://localhost:25000/varz?json"
+VARZ_HTTPS_URL = "https://localhost:25000/varz?json"
+
+
+@CustomClusterTestSuite.with_args(cluster_size=1,
+ catalogd_args="--avro_schema_url_remote_http_enabled=false")
+class TestAvroSchemaUrlHttpDisabled(CustomClusterTestSuite):
+ """HTTP schema fetching is rejected when avro_schema_url_remote_http_enabled=false
+ (the default)."""
+
+ @pytest.mark.parametrize("url", [VARZ_HTTP_URL, VARZ_HTTPS_URL])
+ def test_http_rejected_when_disabled(self, unique_name, url):
+ """CREATE TABLE with an HTTP avro.schema.url fails with the SSRF-guard error."""
+ try:
+ result = self.execute_query_expect_failure(
+ self.client,
+ "create table default.{t} stored as avro tblproperties "
+ "('avro.schema.url'='{url}')".format(t=unique_name, url=url))
+ assert HTTP_DISABLED_ERROR in str(result)
+ finally:
+ self.client.execute(
+ "drop table if exists default.{t}".format(t=unique_name))
+
+
+@CustomClusterTestSuite.with_args(cluster_size=1,
+ catalogd_args=("--avro_schema_url_remote_http_enabled=true "
+ "--avro_schema_url_http_allowed_hosts=localhost"))
+class TestAvroSchemaUrlHttpEnabled(CustomClusterTestSuite):
+ """HTTP schema fetching is gated on host allowlist when
+ avro_schema_url_remote_http_enabled=true."""
+
+ @pytest.mark.parametrize("url,exc", [(VARZ_HTTP_URL, 'SchemaParseException'),
+ (VARZ_HTTPS_URL, 'SSLHandshakeException')])
+ def test_allowed_host_fetch_attempted(self, unique_name, url, exc):
+ """A URL whose host is in avro_schema_url_http_allowed_hosts is not blocked
+ by the flag guard. localhost:25000/varz?json returns valid JSON but not a
+ valid Avro schema, so the DDL fails at schema-parse time, not at the flag
+ check."""
+ try:
+ result = self.execute_query_expect_failure(
+ self.client,
+ "create table default.{t} stored as avro tblproperties "
+ "('avro.schema.url'='{url}')".format(t=unique_name, url=url))
+ assert HTTP_DISABLED_ERROR not in str(result)
+ assert HOST_NOT_ALLOWED_ERROR not in str(result)
+ assert exc in str(result)
+ finally:
+ self.client.execute(
+ "drop table if exists default.{t}".format(t=unique_name))
+
+ @pytest.mark.parametrize("url", ["http://external.example.com/schema.json",
+ "https://external.example.com/schema.json"])
+ def test_unlisted_host_rejected(self, unique_name, url):
+ """A URL whose host is absent from avro_schema_url_http_allowed_hosts is
+ rejected by the flag guard."""
+ try:
+ result = self.execute_query_expect_failure(
+ self.client,
+ "create table default.{t} stored as avro tblproperties "
+ "('avro.schema.url'='{url}')".format(t=unique_name, url=url))
+ assert HOST_NOT_ALLOWED_ERROR in str(result)
+ finally:
+ self.client.execute(
+ "drop table if exists default.{t}".format(t=unique_name))
+
+
+@CustomClusterTestSuite.with_args(cluster_size=1,
+ catalogd_args="--avro_schema_url_allowed_schemes=")
+class TestAvroSchemaUrlAllowedSchemes(CustomClusterTestSuite):
+ """avro_schema_url_allowed_schemes restricts which URI schemes may be fetched."""
+
+ def test_unlisted_scheme_rejected(self, unique_name):
+ """A URI whose scheme is absent from avro_schema_url_allowed_schemes is
+ rejected by the flag guard."""
+ fs_url = FILESYSTEM_URI_SCHEME + ":///nonexistent_path/schema.json"
+ try:
+ result = self.execute_query_expect_failure(
+ self.client,
+ "create table default.{t} stored as avro tblproperties "
+ "('avro.schema.url'='{url}')".format(t=unique_name, url=fs_url))
+ assert SCHEME_NOT_ALLOWED_ERROR in str(result)
+ finally:
+ self.client.execute(
+ "drop table if exists default.{t}".format(t=unique_name))
+
+ def test_empty_scheme_uses_default_fs(self, unique_name):
+ """A URI with no scheme uses the default FS, and is always allowed.
+ The path does not exist so the DDL fails, but not due to the scheme check."""
+ fs_url = "/nonexistent_path/schema.json"
+ try:
+ result = self.execute_query_expect_failure(
+ self.client,
+ "create table default.{t} stored as avro tblproperties "
+ "('avro.schema.url'='{url}')".format(t=unique_name, url=fs_url))
+ assert SCHEME_NOT_ALLOWED_ERROR not in str(result)
+ assert 'FileNotFoundException' in str(result)
+ finally:
+ self.client.execute(
+ "drop table if exists default.{t}".format(t=unique_name))