You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ji...@apache.org on 2021/04/13 16:49:05 UTC
[druid] branch 0.21.0 updated: Enforce allow list for JDBC
properties by default (#11063) (#11107)
This is an automated email from the ASF dual-hosted git repository.
jihoonson pushed a commit to branch 0.21.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/0.21.0 by this push:
new 8efac3d Enforce allow list for JDBC properties by default (#11063) (#11107)
8efac3d is described below
commit 8efac3da28ca69a9792dde1daf8d1a9b75b4b873
Author: Jihoon Son <ji...@apache.org>
AuthorDate: Tue Apr 13 09:48:37 2021 -0700
Enforce allow list for JDBC properties by default (#11063) (#11107)
* Enforce allow list for JDBC properties by default
* fix tests
---
docs/configuration/index.md | 24 +++++++++++++++++++++-
.../sql/MySQLFirehoseDatabaseConnectorTest.java | 13 ++++++++----
.../PostgresqlFirehoseDatabaseConnectorTest.java | 6 ++++++
.../initialization/JdbcAccessSecurityConfig.java | 9 ++++----
.../ExternalStorageAccessSecurityModuleTest.java | 2 +-
5 files changed, 43 insertions(+), 11 deletions(-)
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 4d192e6..702788b 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -517,6 +517,28 @@ This deep storage is used to interface with Cassandra. Note that the `druid-cas
### Ingestion Security Configuration
+#### HDFS input source
+
+You can set the following property to specify permissible protocols for
+the [HDFS input source](../ingestion/native-batch.md#hdfs-input-source) and the [HDFS firehose](../ingestion/native-batch.md#hdfsfirehose).
+
+|Property|Possible Values|Description|Default|
+|--------|---------------|-----------|-------|
+|`druid.ingestion.hdfs.allowedProtocols`|List of protocols|Allowed protocols for the HDFS input source and HDFS firehose.|["hdfs"]|
+
+
+#### HTTP input source
+
+You can set the following property to specify permissible protocols for
+the [HTTP input source](../ingestion/native-batch.md#http-input-source) and the [HTTP firehose](../ingestion/native-batch.md#httpfirehose).
+
+|Property|Possible Values|Description|Default|
+|--------|---------------|-----------|-------|
+|`druid.ingestion.http.allowedProtocols`|List of protocols|Allowed protocols for the HTTP input source and HTTP firehose.|["http", "https"]|
+
+
+### External Data Access Security Configuration
+
#### JDBC Connections to External Databases
You can use the following properties to specify permissible JDBC options for:
@@ -529,7 +551,7 @@ These properties do not apply to metadata storage connections.
|Property|Possible Values|Description|Default|
|--------|---------------|-----------|-------|
-|`druid.access.jdbc.enforceAllowedProperties`|Boolean|When true, Druid applies `druid.access.jdbc.allowedProperties` to JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:`. When false, Druid allows any kind of JDBC connections without JDBC property validation. This config is deprecated and will be removed in a future release.|false|
+|`druid.access.jdbc.enforceAllowedProperties`|Boolean|When true, Druid applies `druid.access.jdbc.allowedProperties` to JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:`. When false, Druid allows any kind of JDBC connections without JDBC property validation. This config is for backward compatibility especially during upgrades since enforcing allow list can break existing ingestion jobs or lookups based on JDBC. This config is deprecated and will be removed in a future re [...]
|`druid.access.jdbc.allowedProperties`|List of JDBC properties|Defines a list of allowed JDBC properties. Druid always enforces the list for all JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:` if `druid.access.jdbc.enforceAllowedProperties` is set to true.<br/><br/>This option is tested against MySQL connector 5.1.48 and PostgreSQL connector 42.2.14. Other connector versions might not work.|["useSSL", "requireSSL", "ssl", "sslmode"]|
|`druid.access.jdbc.allowUnknownJdbcUrlFormat`|Boolean|When false, Druid only accepts JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:`. When true, Druid allows JDBC connections to any kind of database, but only enforces `druid.access.jdbc.allowedProperties` for PostgreSQL and MySQL.|true|
diff --git a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/firehose/sql/MySQLFirehoseDatabaseConnectorTest.java b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/firehose/sql/MySQLFirehoseDatabaseConnectorTest.java
index 4778a42..150f3ca 100644
--- a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/firehose/sql/MySQLFirehoseDatabaseConnectorTest.java
+++ b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/firehose/sql/MySQLFirehoseDatabaseConnectorTest.java
@@ -184,6 +184,12 @@ public class MySQLFirehoseDatabaseConnectorTest
{
return ImmutableSet.of("user", "nonenone");
}
+
+ @Override
+ public boolean isEnforceAllowedProperties()
+ {
+ return false;
+ }
};
new MySQLFirehoseDatabaseConnector(
@@ -205,13 +211,12 @@ public class MySQLFirehoseDatabaseConnectorTest
}
};
- MySQLFirehoseDatabaseConnector connector = new MySQLFirehoseDatabaseConnector(
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage(StringUtils.format("Invalid URL format for MySQL: [%s]", url));
+ new MySQLFirehoseDatabaseConnector(
connectorConfig,
new JdbcAccessSecurityConfig()
);
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage(StringUtils.format("Invalid URL format for MySQL: [%s]", url));
- connector.findPropertyKeysFromConnectURL(url);
}
private static JdbcAccessSecurityConfig newSecurityConfigEnforcingAllowList(Set<String> allowedProperties)
diff --git a/extensions-core/postgresql-metadata-storage/src/test/java/org/apache/druid/firehose/PostgresqlFirehoseDatabaseConnectorTest.java b/extensions-core/postgresql-metadata-storage/src/test/java/org/apache/druid/firehose/PostgresqlFirehoseDatabaseConnectorTest.java
index 4ab9ab3..b1a50bb 100644
--- a/extensions-core/postgresql-metadata-storage/src/test/java/org/apache/druid/firehose/PostgresqlFirehoseDatabaseConnectorTest.java
+++ b/extensions-core/postgresql-metadata-storage/src/test/java/org/apache/druid/firehose/PostgresqlFirehoseDatabaseConnectorTest.java
@@ -183,6 +183,12 @@ public class PostgresqlFirehoseDatabaseConnectorTest
{
return ImmutableSet.of("user", "nonenone");
}
+
+ @Override
+ public boolean isEnforceAllowedProperties()
+ {
+ return false;
+ }
};
new PostgresqlFirehoseDatabaseConnector(
diff --git a/server/src/main/java/org/apache/druid/server/initialization/JdbcAccessSecurityConfig.java b/server/src/main/java/org/apache/druid/server/initialization/JdbcAccessSecurityConfig.java
index ba12ec0..f2eda2c 100644
--- a/server/src/main/java/org/apache/druid/server/initialization/JdbcAccessSecurityConfig.java
+++ b/server/src/main/java/org/apache/druid/server/initialization/JdbcAccessSecurityConfig.java
@@ -69,13 +69,12 @@ public class JdbcAccessSecurityConfig
@JsonProperty
private boolean allowUnknownJdbcUrlFormat = true;
- // Enforcing allow list check can break rolling upgrade. This is not good for patch releases
- // and is why this config is added. However, from the security point of view, this config
- // should be always enabled in production to secure your cluster. As a result, this config
- // is deprecated and will be removed in the near future.
+ // This config is for compatibility as enforcing allow list can break existing ingestion jobs or lookups.
+ // However, from the security point of view, this config should be always enabled in production to secure
+ // your cluster. As a result, this config is deprecated and will be removed in a future release.
@Deprecated
@JsonProperty
- private boolean enforceAllowedProperties = false;
+ private boolean enforceAllowedProperties = true;
@JsonIgnore
public Set<String> getSystemPropertyPrefixes()
diff --git a/server/src/test/java/org/apache/druid/server/initialization/ExternalStorageAccessSecurityModuleTest.java b/server/src/test/java/org/apache/druid/server/initialization/ExternalStorageAccessSecurityModuleTest.java
index 7f092dd..27f54b7 100644
--- a/server/src/test/java/org/apache/druid/server/initialization/ExternalStorageAccessSecurityModuleTest.java
+++ b/server/src/test/java/org/apache/druid/server/initialization/ExternalStorageAccessSecurityModuleTest.java
@@ -47,7 +47,7 @@ public class ExternalStorageAccessSecurityModuleTest
securityConfig.getAllowedProperties()
);
Assert.assertTrue(securityConfig.isAllowUnknownJdbcUrlFormat());
- Assert.assertFalse(securityConfig.isEnforceAllowedProperties());
+ Assert.assertTrue(securityConfig.isEnforceAllowedProperties());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org