You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2021/05/14 22:08:21 UTC
[airflow] branch master updated: Default `resultBackendConnection`
to `metadataConnection` (#15861)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 2180608 Default `resultBackendConnection` to `metadataConnection` (#15861)
2180608 is described below
commit 218060882f6071f5cbbefbe87fb98ff4d8d3d240
Author: Jed Cunningham <66...@users.noreply.github.com>
AuthorDate: Fri May 14 16:08:06 2021 -0600
Default `resultBackendConnection` to `metadataConnection` (#15861)
Instead of requiring anyone using an external db and CeleryExecutor to
set their database details twice, let the default for
`resultBackendConnection` be the values from `metadataConnection`.
Anyone who wants to use a separate backend for results still can.
---
chart/templates/_helpers.yaml | 8 ++--
.../secrets/result-backend-connection-secret.yaml | 11 ++---
chart/tests/test_pgbouncer.py | 10 ++++-
.../tests/test_result_backend_connection_secret.py | 47 ++++++++++++++++++----
chart/values.schema.json | 29 +++++++++----
chart/values.yaml | 19 +++++----
6 files changed, 91 insertions(+), 33 deletions(-)
diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index bc154ac..8fd7bf0 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -304,11 +304,12 @@ If release name contains chart name it will be used as a full name.
{{- end }}
{{ define "pgbouncer_config" }}
+{{- $resultBackendConnection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }}
{{- $pgMetadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }}
-{{- $pgResultBackendHost := .Values.data.resultBackendConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }}
+{{- $pgResultBackendHost := $resultBackendConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }}
[databases]
{{ .Release.Name }}-metadata = host={{ $pgMetadataHost }} dbname={{ .Values.data.metadataConnection.db }} port={{ .Values.data.metadataConnection.port }} pool_size={{ .Values.pgbouncer.metadataPoolSize }}
-{{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ .Values.data.resultBackendConnection.db }} port={{ .Values.data.resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }}
+{{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ $resultBackendConnection.db }} port={{ $resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }}
[pgbouncer]
pool_mode = transaction
@@ -339,8 +340,9 @@ server_tls_key_file = /etc/pgbouncer/server.key
{{- end }}
{{ define "pgbouncer_users" }}
+{{- $resultBackendConnection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }}
{{ .Values.data.metadataConnection.user | quote }} {{ .Values.data.metadataConnection.pass | quote }}
-{{ .Values.data.resultBackendConnection.user | quote }} {{ .Values.data.resultBackendConnection.pass | quote }}
+{{ $resultBackendConnection.user | quote }} {{ $resultBackendConnection.pass | quote }}
{{- end }}
{{ define "airflow_logs" -}}
diff --git a/chart/templates/secrets/result-backend-connection-secret.yaml b/chart/templates/secrets/result-backend-connection-secret.yaml
index cafd6e9..c773972 100644
--- a/chart/templates/secrets/result-backend-connection-secret.yaml
+++ b/chart/templates/secrets/result-backend-connection-secret.yaml
@@ -20,13 +20,14 @@
#################################
{{- if not .Values.data.resultBackendSecretName }}
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
+{{- $connection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }}
-{{- $resultBackendHost := .Values.data.resultBackendConnection.host | default (printf "%s-%s" .Release.Name "postgresql") }}
+{{- $resultBackendHost := $connection.host | default (printf "%s-%s" .Release.Name "postgresql") }}
{{- $pgbouncerHost := printf "%s-%s" .Release.Name "pgbouncer" }}
{{- $host := ternary $pgbouncerHost $resultBackendHost .Values.pgbouncer.enabled }}
-{{- $port := (ternary .Values.ports.pgbouncer .Values.data.resultBackendConnection.port .Values.pgbouncer.enabled) | toString }}
-{{- $database := ternary (printf "%s-%s" .Release.Name "result-backend") .Values.data.resultBackendConnection.db .Values.pgbouncer.enabled }}
-{{- $extras := ternary (printf "?sslmode=%s" .Values.data.resultBackendConnection.sslmode) "" (eq .Values.data.resultBackendConnection.protocol "postgresql") }}
+{{- $port := (ternary .Values.ports.pgbouncer $connection.port .Values.pgbouncer.enabled) | toString }}
+{{- $database := ternary (printf "%s-%s" .Release.Name "result-backend") $connection.db .Values.pgbouncer.enabled }}
+{{- $extras := ternary (printf "?sslmode=%s" $connection.sslmode) "" (eq $connection.protocol "postgresql") }}
kind: Secret
apiVersion: v1
metadata:
@@ -41,6 +42,6 @@ metadata:
{{- end }}
type: Opaque
data:
- connection: {{ (printf "db+%s://%s:%s@%s:%s/%s%s" .Values.data.resultBackendConnection.protocol .Values.data.resultBackendConnection.user .Values.data.resultBackendConnection.pass $host $port $database $extras) | b64enc | quote }}
+ connection: {{ (printf "db+%s://%s:%s@%s:%s/%s%s" $connection.protocol $connection.user $connection.pass $host $port $database $extras) | b64enc | quote }}
{{- end }}
{{- end }}
diff --git a/chart/tests/test_pgbouncer.py b/chart/tests/test_pgbouncer.py
index 7677a7b..47f9370 100644
--- a/chart/tests/test_pgbouncer.py
+++ b/chart/tests/test_pgbouncer.py
@@ -230,7 +230,15 @@ class PgbouncerConfigTest(unittest.TestCase):
"pgbouncer": {"enabled": True, "metadataPoolSize": 12, "resultBackendPoolSize": 7},
"data": {
"metadataConnection": {"host": "meta_host", "db": "meta_db", "port": 1111},
- "resultBackendConnection": {"host": "rb_host", "db": "rb_db", "port": 2222},
+ "resultBackendConnection": {
+ "protocol": "postgresql",
+ "host": "rb_host",
+ "user": "someuser",
+ "pass": "someuser",
+ "db": "rb_db",
+ "port": 2222,
+ "sslmode": "disabled",
+ },
},
}
ini = self._get_pgbouncer_ini(values)
diff --git a/chart/tests/test_result_backend_connection_secret.py b/chart/tests/test_result_backend_connection_secret.py
index 74b61b6..2c1c684 100644
--- a/chart/tests/test_result_backend_connection_secret.py
+++ b/chart/tests/test_result_backend_connection_secret.py
@@ -30,8 +30,10 @@ class ResultBackendConnectionSecretTest(unittest.TestCase):
"user": "someuser",
"pass": "somepass",
"host": "somehost",
+ "protocol": "postgresql",
"port": 7777,
"db": "somedb",
+ "sslmode": "allow",
}
def test_should_not_generate_a_document_if_using_existing_secret(self):
@@ -73,6 +75,19 @@ class ResultBackendConnectionSecretTest(unittest.TestCase):
== connection
)
+ def test_should_default_to_custom_metadata_db_connection_with_pgbouncer_overrides(self):
+ values = {
+ "pgbouncer": {"enabled": True},
+ "data": {"metadataConnection": {**self.non_chart_database_values}},
+ }
+ connection = self._get_connection(values)
+
+ # host, port, dbname still get overridden
+ assert (
+ "db+postgresql://someuser:somepass@RELEASE-NAME-pgbouncer:6543"
+ "/RELEASE-NAME-result-backend?sslmode=allow" == connection
+ )
+
def test_should_set_pgbouncer_overrides_when_enabled(self):
values = {"pgbouncer": {"enabled": True}}
connection = self._get_connection(values)
@@ -93,32 +108,48 @@ class ResultBackendConnectionSecretTest(unittest.TestCase):
# host, port, dbname still get overridden even with an non-chart db
assert (
"db+postgresql://someuser:somepass@RELEASE-NAME-pgbouncer:6543"
- "/RELEASE-NAME-result-backend?sslmode=disable" == connection
+ "/RELEASE-NAME-result-backend?sslmode=allow" == connection
)
+ def test_should_default_to_custom_metadata_db_connection(self):
+ values = {
+ "data": {"metadataConnection": {**self.non_chart_database_values}},
+ }
+ connection = self._get_connection(values)
+
+ assert "db+postgresql://someuser:somepass@somehost:7777/somedb?sslmode=allow" == connection
+
def test_should_correctly_use_non_chart_database(self):
+ values = {"data": {"resultBackendConnection": {**self.non_chart_database_values}}}
+ connection = self._get_connection(values)
+
+ assert "db+postgresql://someuser:somepass@somehost:7777/somedb?sslmode=allow" == connection
+
+ def test_should_support_non_postgres_db(self):
values = {
"data": {
"resultBackendConnection": {
**self.non_chart_database_values,
- "sslmode": "require",
+ "protocol": "mysql",
}
}
}
connection = self._get_connection(values)
- assert "db+postgresql://someuser:somepass@somehost:7777/somedb?sslmode=require" == connection
+ # sslmode is only added for postgresql
+ assert "db+mysql://someuser:somepass@somehost:7777/somedb" == connection
- def test_should_support_non_postgres_db(self):
+ def test_should_correctly_use_non_chart_database_when_both_db_are_external(self):
values = {
"data": {
+ "metadataConnection": {**self.non_chart_database_values},
"resultBackendConnection": {
**self.non_chart_database_values,
- "protocol": "mysql",
- }
+ "user": "anotheruser",
+ "pass": "anotherpass",
+ },
}
}
connection = self._get_connection(values)
- # sslmode is only added for postgresql
- assert "db+mysql://someuser:somepass@somehost:7777/somedb" == connection
+ assert "db+postgresql://anotheruser:anotherpass@somehost:7777/somedb?sslmode=allow" == connection
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 6625858..e5b4a11 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -690,23 +690,27 @@
},
"resultBackendConnection": {
"description": "Result backend connection configuration.",
- "type": "object",
+ "type": [
+ "object",
+ "null"
+ ],
+ "default": null,
"additionalProperties": false,
"properties": {
"user": {
"description": "The database user.",
"type": "string",
- "default": "postgres"
+ "default": null
},
"pass": {
"description": "The database password.",
"type": "string",
- "default": "postgres"
+ "default": null
},
"protocol": {
"description": "The database protocol.",
"type": "string",
- "default": "postgresql"
+ "default": null
},
"host": {
"description": "The database host.",
@@ -719,19 +723,28 @@
"port": {
"description": "The database port.",
"type": "integer",
- "default": 5432
+ "default": null
},
"db": {
"description": "The name of the database.",
"type": "string",
- "default": "postgres"
+ "default": null
},
"sslmode": {
"description": "The database SSL parameter.",
"type": "string",
- "default": "disable"
+ "default": null
}
- }
+ },
+ "required": [
+ "user",
+ "pass",
+ "protocol",
+ "host",
+ "port",
+ "db",
+ "sslmode"
+ ]
},
"brokerUrl": {
"description": "Direct url to the redis broker (when using an external redis instance).",
diff --git a/chart/values.yaml b/chart/values.yaml
index a89853a..e7e5533 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -249,14 +249,17 @@ data:
port: 5432
db: postgres
sslmode: disable
- resultBackendConnection:
- user: postgres
- pass: postgres
- protocol: postgresql
- host: ~
- port: 5432
- db: postgres
- sslmode: disable
+ # resultBackendConnection defaults to the same database as metadataConnection
+ resultBackendConnection: ~
+ # or, you can use a different database
+ # resultBackendConnection:
+ # user: postgres
+ # pass: postgres
+ # protocol: postgresql
+ # host: ~
+ # port: 5432
+ # db: postgres
+ # sslmode: disable
brokerUrl: ~
# Fernet key settings