You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/07/12 13:02:37 UTC
[incubator-datalab] branch DATALAB-2890 created (now 4a404e40e)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2890
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
at 4a404e40e [DATALAB-2890] superset version update changes
This branch includes the following new commits:
new feb2575a7 [DATALAB-2890] superset version update changes
new 4a404e40e [DATALAB-2890] superset version update changes
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/02: [DATALAB-2890] superset version update changes
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2890
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 4a404e40e38d6f5dfb57396300941d925923aa98
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Jul 12 13:07:58 2022 +0300
[DATALAB-2890] superset version update changes
---
infrastructure-provisioning/src/general/lib/os/fab.py | 1 +
infrastructure-provisioning/src/superset/templates/superset_config.py | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index eac1b2fa5..6635258d2 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -1346,6 +1346,7 @@ def configure_superset(os_user, keycloak_auth_server_url, keycloak_realm_name, k
keycloak_auth_server_url))
conn.sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/datalab/templates/superset_config.py'.format(
keycloak_realm_name))
+ conn.sudo('sed -i \'s/OS_USER/{}/g\' /opt/datalab/templates/superset_config.py'.format(os_user))
conn.sudo(
'sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/superset_config.py'.format(edge_instance_public_ip))
conn.sudo('sed -i \'s/SUPERSET_NAME/{}/g\' /opt/datalab/templates/superset_config.py'.format(superset_name))
diff --git a/infrastructure-provisioning/src/superset/templates/superset_config.py b/infrastructure-provisioning/src/superset/templates/superset_config.py
index f30a043f4..358e4b438 100644
--- a/infrastructure-provisioning/src/superset/templates/superset_config.py
+++ b/infrastructure-provisioning/src/superset/templates/superset_config.py
@@ -119,7 +119,7 @@ AUTH_TYPE = AUTH_OID
AUTH_USER_REGISTRATION = True
AUTH_USER_REGISTRATION_ROLE = "Admin"
CUSTOM_SECURITY_MANAGER = SupersetOIDCSecurityManager
-OIDC_CLIENT_SECRETS = '/home/datalab-user/superset/docker/id_provider.json'
+OIDC_CLIENT_SECRETS = '/home/OS_USER/superset/docker/id_provider.json'
OIDC_COOKIE_SECURE = False
OIDC_VALID_ISSUERS = 'KEYCLOAK_AUTH_SERVER_URL/realms/KEYCLOAK_REALM_NAME'
WTF_CSRF_ENABLED = False
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/02: [DATALAB-2890] superset version update changes
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2890
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit feb2575a7bdb8eeb7e31a991e43d273c4729bbb0
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Jul 12 12:47:36 2022 +0300
[DATALAB-2890] superset version update changes
---
.../src/general/conf/datalab.ini | 2 +-
.../src/general/lib/os/fab.py | 24 ++--
.../src/general/scripts/os/superset_start.py | 4 +-
.../templates/os/debian/superset-notebook.service | 2 +-
.../src/superset/templates/docker-compose.yml | 148 ++++++++++++++++-----
.../src/superset/templates/docker-init.sh | 58 ++++++--
.../templates/keycloak_security_manager.py | 71 ++++++++++
.../src/superset/templates/requirements-extra.txt | 1 +
.../src/superset/templates/superset_config.py | 103 ++++++++++----
9 files changed, 334 insertions(+), 79 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 0b5277b52..f6c4e92a8 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -357,7 +357,7 @@ matplotlib_version = 3.3.4
### JupyterLab image
jupyterlab_image = odahu\/base-notebook:1.1.0-rc8
### Superset version
-superset_version = 0.35.1
+superset_version = 1.5.1
### GCS-connector version
gcs_connector_version = 2.0.1
### Setuptools version
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index add3157c3..eac1b2fa5 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -1319,18 +1319,17 @@ def configure_superset(os_user, keycloak_auth_server_url, keycloak_realm_name, k
keycloak_client_secret, edge_instance_private_ip, edge_instance_public_ip, superset_name):
logging.info('Superset configuring')
try:
- if not exists(conn, '/home/{}/incubator-superset'.format(os_user)):
+ if not exists(conn, '/home/{}/superset'.format(os_user)):
conn.sudo(
'''bash -c 'cd /home/{} && wget https://github.com/apache/incubator-superset/archive/{}.tar.gz' '''.format(
os_user, os.environ['notebook_superset_version']))
conn.sudo('''bash -c 'cd /home/{} && tar -xzf {}.tar.gz' '''.format(os_user, os.environ[
'notebook_superset_version']))
- conn.sudo('''bash -c 'cd /home/{} && ln -sf incubator-superset-{} incubator-superset' '''.format(os_user,
- os.environ[
- 'notebook_superset_version']))
+ conn.sudo('''bash -c 'cd /home/{} && ln -sf superset-{} superset' '''.format(os_user,
+ os.environ['notebook_superset_version']))
if not exists(conn, '/tmp/superset-notebook_installed'):
conn.sudo('mkdir -p /opt/datalab/templates')
- conn.local('cd /root/templates; tar -zcvf /tmp/templates.tar.gz *')
+ conn.local('cd /root/templates; tar -zcvf /tmp/templates.tar.gz .')
conn.put('/tmp/templates.tar.gz', '/tmp/templates.tar.gz')
conn.sudo('tar -zxvf /tmp/templates.tar.gz -C /opt/datalab/templates')
conn.sudo('sed -i \'s/OS_USER/{}/g\' /opt/datalab/templates/.env'.format(os_user))
@@ -1350,20 +1349,23 @@ def configure_superset(os_user, keycloak_auth_server_url, keycloak_realm_name, k
conn.sudo(
'sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/superset_config.py'.format(edge_instance_public_ip))
conn.sudo('sed -i \'s/SUPERSET_NAME/{}/g\' /opt/datalab/templates/superset_config.py'.format(superset_name))
- conn.sudo('cp -f /opt/datalab/templates/.env /home/{}/incubator-superset/contrib/docker/'.format(os_user))
+ conn.sudo('cp -f /opt/datalab/templates/.env /home/{}/superset/docker/'.format(os_user))
conn.sudo(
- 'cp -f /opt/datalab/templates/docker-compose.yml /home/{}/incubator-superset/contrib/docker/'.format(
+ 'cp -f /opt/datalab/templates/docker-compose.yml /home/{}/superset/'.format(
os_user))
conn.sudo(
- 'cp -f /opt/datalab/templates/id_provider.json /home/{}/incubator-superset/contrib/docker/'.format(
+ 'cp -f /opt/datalab/templates/id_provider.json /home/{}/superset/docker/'.format(
os_user))
conn.sudo(
- 'cp -f /opt/datalab/templates/requirements-extra.txt /home/{}/incubator-superset/contrib/docker/'.format(
+ 'cp -f /opt/datalab/templates/requirements-extra.txt /home/{}/superset/requirements/'.format(
os_user))
conn.sudo(
- 'cp -f /opt/datalab/templates/superset_config.py /home/{}/incubator-superset/contrib/docker/'.format(
+ 'cp -f /opt/datalab/templates/superset_config.py /home/{}/superset/docker/pythonpath_dev/'.format(
os_user))
- conn.sudo('cp -f /opt/datalab/templates/docker-init.sh /home/{}/incubator-superset/contrib/docker/'.format(
+ conn.sudo(
+ 'cp -f /opt/datalab/templates/keycloak_security_manager.py /home/{}/superset/docker/pythonpath_dev/'.format(
+ os_user))
+ conn.sudo('cp -f /opt/datalab/templates/docker-init.sh /home/{}/superset/docker/'.format(
os_user))
conn.sudo('touch /tmp/superset-notebook_installed')
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/scripts/os/superset_start.py b/infrastructure-provisioning/src/general/scripts/os/superset_start.py
index 9f976ae8d..924512883 100644
--- a/infrastructure-provisioning/src/general/scripts/os/superset_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/superset_start.py
@@ -33,11 +33,11 @@ parser.add_argument('--keyfile', type=str, default='')
parser.add_argument('--os_user', type=str, default='')
args = parser.parse_args()
-superset_dir = '/home/' + args.os_user + '/incubator-superset/contrib/docker'
+superset_dir = '/home/' + args.os_user + '/superset'
def start_superset(superset_dir):
try:
- conn.sudo('''bash -c 'cd {} && docker-compose run --rm superset ./docker-init.sh' '''.format(superset_dir))
+ conn.sudo('''bash -c 'cd {} && docker-compose run --rm superset docker/docker-init.sh' '''.format(superset_dir))
conn.sudo('cp /opt/datalab/templates/superset-notebook.service /tmp/')
conn.sudo('sed -i \'s/OS_USER/{}/g\' /tmp/superset-notebook.service'.format(args.os_user))
conn.sudo('cp /tmp/superset-notebook.service /etc/systemd/system/')
diff --git a/infrastructure-provisioning/src/general/templates/os/debian/superset-notebook.service b/infrastructure-provisioning/src/general/templates/os/debian/superset-notebook.service
index ef3b4bf14..064a8b777 100644
--- a/infrastructure-provisioning/src/general/templates/os/debian/superset-notebook.service
+++ b/infrastructure-provisioning/src/general/templates/os/debian/superset-notebook.service
@@ -28,7 +28,7 @@ User=OS_USER
Group=OS_USER
ExecStart=/usr/bin/sudo docker-compose up
ExecStop=/usr/bin/sudo docker-compose stop
-WorkingDirectory=/home/OS_USER/incubator-superset/contrib/docker
+WorkingDirectory=/home/OS_USER/superset
TimeoutStopSec=120
Restart=on-failure
RestartSec=10
diff --git a/infrastructure-provisioning/src/superset/templates/docker-compose.yml b/infrastructure-provisioning/src/superset/templates/docker-compose.yml
index 78283cb67..5fe5609c8 100644
--- a/infrastructure-provisioning/src/superset/templates/docker-compose.yml
+++ b/infrastructure-provisioning/src/superset/templates/docker-compose.yml
@@ -14,18 +14,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-version: '2'
+x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
+x-superset-user: &superset-user root
+x-superset-depends-on: &superset-depends-on
+ - db
+ - redis
+x-superset-volumes: &superset-volumes
+ # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
+ - ./docker:/app/docker
+ - ./superset:/app/superset
+ - ./superset-frontend:/app/superset-frontend
+ - superset_home:/app/superset_home
+ - ./tests:/app/tests
+
+version: "3.7"
services:
redis:
- image: redis:3.2
+ image: redis:latest
+ container_name: superset_cache
restart: unless-stopped
ports:
- "127.0.0.1:6379:6379"
volumes:
- redis:/data
- postgres:
- image: postgres:10
+ db:
+ env_file: docker/.env
+ image: postgres:14
+ container_name: superset_db
restart: unless-stopped
environment:
POSTGRES_DB: superset
@@ -34,44 +50,112 @@ services:
ports:
- "127.0.0.1:5432:5432"
volumes:
- - postgres:/var/lib/postgresql/data
+ - db_home:/var/lib/postgresql/data
superset:
- build:
- context: ../../
- dockerfile: contrib/docker/Dockerfile
+ env_file: docker/.env
+ image: *superset-image
+ container_name: superset_app
+ command: ["/app/docker/docker-bootstrap.sh", "app"]
restart: unless-stopped
- environment:
- POSTGRES_DB: superset
- POSTGRES_USER: superset
- POSTGRES_PASSWORD: superset
- POSTGRES_HOST: postgres
- POSTGRES_PORT: 5432
- REDIS_HOST: redis
- REDIS_PORT: 6379
- USERNAME_OIDC_FIELD: preferred_username
- FIRST_NAME_OIDC_FIELD: given_name
- LAST_NAME_OIDC_FIELD: family_name
- http_proxy: http://PROXY_STRING
- https_proxy: http://PROXY_STRING
- # If using production, comment development volume below
- SUPERSET_ENV: production
- #SUPERSET_ENV: development
- user: root:root
ports:
- 8088:8088
+ user: *superset-user
+ depends_on: *superset-depends-on
+ volumes: *superset-volumes
+ environment:
+ CYPRESS_CONFIG: "${CYPRESS_CONFIG}"
+
+ superset-websocket:
+ container_name: superset_websocket
+ build: ./superset-websocket
+ image: superset-websocket
+ ports:
+ - 8080:8080
depends_on:
- - postgres
- redis
+ # Mount everything in superset-websocket into container and
+ # then exclude node_modules and dist with bogus volume mount.
+ # This is necessary because host and container need to have
+ # their own, separate versions of these files. .dockerignore
+ # does not seem to work when starting the service through
+ # docker-compose.
+ #
+ # For example, node_modules may contain libs with native bindings.
+ # Those bindings need to be compiled for each OS and the container
+ # OS is not necessarily the same as host OS.
volumes:
- # this is needed to communicate with the postgres and redis services
- - ./superset_config.py:/home/superset/superset/superset_config.py
- # this is needed for development, remove with SUPERSET_ENV=production
- #- ../../superset:/home/superset/superset
- - ./id_provider.json:/home/superset/superset/id_provider.json
+ - ./superset-websocket:/home/superset-websocket
+ - /home/superset-websocket/node_modules
+ - /home/superset-websocket/dist
+ environment:
+ - PORT=8080
+ - REDIS_HOST=redis
+ - REDIS_PORT=6379
+ - REDIS_SSL=false
+
+ superset-init:
+ image: *superset-image
+ container_name: superset_init
+ command: ["/app/docker/docker-init.sh"]
+ env_file: docker/.env
+ depends_on: *superset-depends-on
+ user: *superset-user
+ volumes: *superset-volumes
+ environment:
+ CYPRESS_CONFIG: "${CYPRESS_CONFIG}"
+
+ superset-node:
+ image: node:16
+ container_name: superset_node
+ command: ["/app/docker/docker-frontend.sh"]
+ env_file: docker/.env
+ depends_on: *superset-depends-on
+ volumes: *superset-volumes
+
+ superset-worker:
+ image: *superset-image
+ container_name: superset_worker
+ command: ["/app/docker/docker-bootstrap.sh", "worker"]
+ env_file: docker/.env
+ restart: unless-stopped
+ depends_on: *superset-depends-on
+ user: *superset-user
+ volumes: *superset-volumes
+ # Bump memory limit if processing selenium / thumbnails on superset-worker
+ # mem_limit: 2038m
+ # mem_reservation: 128M
+
+ superset-worker-beat:
+ image: *superset-image
+ container_name: superset_worker_beat
+ command: ["/app/docker/docker-bootstrap.sh", "beat"]
+ env_file: docker/.env
+ restart: unless-stopped
+ depends_on: *superset-depends-on
+ user: *superset-user
+ volumes: *superset-volumes
+
+ superset-tests-worker:
+ image: *superset-image
+ container_name: superset_tests_worker
+ command: ["/app/docker/docker-bootstrap.sh", "worker"]
+ env_file: docker/.env
+ environment:
+ DATABASE_HOST: localhost
+ DATABASE_DB: test
+ REDIS_CELERY_DB: 2
+ REDIS_RESULTS_DB: 3
+ REDIS_HOST: localhost
+ network_mode: host
+ depends_on: *superset-depends-on
+ user: *superset-user
+ volumes: *superset-volumes
volumes:
- postgres:
+ superset_home:
+ external: false
+ db_home:
external: false
redis:
external: false
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/superset/templates/docker-init.sh b/infrastructure-provisioning/src/superset/templates/docker-init.sh
index 11b682a91..d5c5d8820 100644
--- a/infrastructure-provisioning/src/superset/templates/docker-init.sh
+++ b/infrastructure-provisioning/src/superset/templates/docker-init.sh
@@ -15,19 +15,59 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-set -ex
+set -e
-# Create an admin user (you will be prompted to set username, first and last name before setting a password)
-export FLASK_APP=superset:app
-#flask fab create-admin --username admin --firstname admin --lastname admin --password admin
+#
+# Always install local overrides first
+#
+/app/docker/docker-bootstrap.sh
-# Initialize the database
-superset db upgrade
+STEP_CNT=4
-if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
- # Load some data to play with
- superset load_examples
+echo_step() {
+cat <<EOF
+######################################################################
+Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
+######################################################################
+EOF
+}
+ADMIN_PASSWORD="admin"
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+ ADMIN_PASSWORD="general"
+ export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
+ export SUPERSET_TESTENV=true
+ export ENABLE_REACT_CRUD_VIEWS=true
+ export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
fi
+# Initialize the database
+echo_step "1" "Starting" "Applying DB migrations"
+superset db upgrade
+echo_step "1" "Complete" "Applying DB migrations"
+# Create an admin user
+echo_step "2" "Starting" "Setting up admin user ( admin / $ADMIN_PASSWORD )"
+superset fab create-admin \
+ --username admin \
+ --firstname Superset \
+ --lastname Admin \
+ --email admin@superset.com \
+ --password $ADMIN_PASSWORD
+echo_step "2" "Complete" "Setting up admin user"
# Create default roles and permissions
+echo_step "3" "Starting" "Setting up roles and perms"
superset init
+echo_step "3" "Complete" "Setting up roles and perms"
+
+if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
+ # Load some data to play with
+ echo_step "4" "Starting" "Loading examples"
+ # If Cypress run which consumes superset_test_config – load required data for tests
+ if [ "$CYPRESS_CONFIG" == "true" ]; then
+ superset load_test_users
+ superset load_examples --load-test-data
+ else
+ superset load_examples
+ fi
+ echo_step "4" "Complete" "Loading examples"
+fi
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/superset/templates/keycloak_security_manager.py b/infrastructure-provisioning/src/superset/templates/keycloak_security_manager.py
new file mode 100644
index 000000000..c7aee93db
--- /dev/null
+++ b/infrastructure-provisioning/src/superset/templates/keycloak_security_manager.py
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This file is included in the final Docker image and SHOULD be overridden when
+# deploying the image to prod. Settings configured here are intended for use in local
+# development environments. Also note that superset_config_docker.py is imported
+# as a final step as a means to override "defaults" configured here
+#
+
+from flask_appbuilder.security.manager import AUTH_OID
+from superset.security import SupersetSecurityManager
+from flask_oidc import OpenIDConnect
+from flask_appbuilder.security.views import AuthOIDView
+from flask_login import login_user
+from urllib.parse import quote
+from flask_appbuilder.views import ModelView, SimpleFormView, expose
+import logging
+
+class OIDCSecurityManager(SupersetSecurityManager):
+
+ def __init__(self, appbuilder):
+ super(OIDCSecurityManager, self).__init__(appbuilder)
+ if self.auth_type == AUTH_OID:
+ self.oid = OpenIDConnect(self.appbuilder.get_app)
+ self.authoidview = AuthOIDCView
+
+class AuthOIDCView(AuthOIDView):
+
+ @expose('/login/', methods=['GET', 'POST'])
+ def login(self, flag=True):
+ sm = self.appbuilder.sm
+ oidc = sm.oid
+
+ @self.appbuilder.sm.oid.require_login
+ def handle_login():
+ user = sm.auth_user_oid(oidc.user_getfield('email'))
+
+ if user is None:
+ info = oidc.user_getinfo(['preferred_username', 'given_name', 'family_name', 'email'])
+ user = sm.add_user(info.get('preferred_username'), info.get('given_name'), info.get('family_name'),
+ info.get('email'), sm.find_role('Gamma'))
+
+ login_user(user, remember=False)
+ return redirect(self.appbuilder.get_url_for_index)
+
+ return handle_login()
+
+ @expose('/logout/', methods=['GET', 'POST'])
+ def logout(self):
+ oidc = self.appbuilder.sm.oid
+
+ oidc.logout()
+ super(AuthOIDCView, self).logout()
+ redirect_url = request.url_root.strip('/') + self.appbuilder.get_url_for_login
+
+ return redirect(
+ oidc.client_secrets.get('issuer') + '/protocol/openid-connect/logout?redirect_uri=' + quote(redirect_url))
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/superset/templates/requirements-extra.txt b/infrastructure-provisioning/src/superset/templates/requirements-extra.txt
index 6b6dcd72f..f337a34fb 100644
--- a/infrastructure-provisioning/src/superset/templates/requirements-extra.txt
+++ b/infrastructure-provisioning/src/superset/templates/requirements-extra.txt
@@ -17,3 +17,4 @@
gevent==1.4.0
fab-oidc
pybigquery
+flask-oidc==1.3.0
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/superset/templates/superset_config.py b/infrastructure-provisioning/src/superset/templates/superset_config.py
index b8830af3e..f30a043f4 100644
--- a/infrastructure-provisioning/src/superset/templates/superset_config.py
+++ b/infrastructure-provisioning/src/superset/templates/superset_config.py
@@ -14,11 +14,26 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# This file is included in the final Docker image and SHOULD be overridden when
+# deploying the image to prod. Settings configured here are intended for use in local
+# development environments. Also note that superset_config_docker.py is imported
+# as a final step as a means to override "defaults" configured here
+#
+import logging
+import os
+from datetime import timedelta
+from typing import Optional
+from keycloak_security_manager import OIDCSecurityManager
+from flask_appbuilder.security.manager import AUTH_OID, AUTH_REMOTE_USER, AUTH_DB, AUTH_LDAP, AUTH_OAUTH
import os
-from fab_oidc.security import SupersetOIDCSecurityManager
-from flask_appbuilder.security.manager import AUTH_OID
+from cachelib.file import FileSystemCache
+from celery.schedules import crontab
-def get_env_variable(var_name, default=None):
+logger = logging.getLogger()
+
+
+def get_env_variable(var_name: str, default: Optional[str] = None) -> str:
"""Get the environment variable or raise exception."""
try:
return os.environ[var_name]
@@ -26,43 +41,85 @@ def get_env_variable(var_name, default=None):
if default is not None:
return default
else:
- error_msg = 'The environment variable {} was missing, abort...'\
- .format(var_name)
+ error_msg = "The environment variable {} was missing, abort...".format(
+ var_name
+ )
raise EnvironmentError(error_msg)
-POSTGRES_USER = get_env_variable('POSTGRES_USER')
-POSTGRES_PASSWORD = get_env_variable('POSTGRES_PASSWORD')
-POSTGRES_HOST = get_env_variable('POSTGRES_HOST')
-POSTGRES_PORT = get_env_variable('POSTGRES_PORT')
-POSTGRES_DB = get_env_variable('POSTGRES_DB')
+DATABASE_DIALECT = get_env_variable("DATABASE_DIALECT")
+DATABASE_USER = get_env_variable("DATABASE_USER")
+DATABASE_PASSWORD = get_env_variable("DATABASE_PASSWORD")
+DATABASE_HOST = get_env_variable("DATABASE_HOST")
+DATABASE_PORT = get_env_variable("DATABASE_PORT")
+DATABASE_DB = get_env_variable("DATABASE_DB")
# The SQLAlchemy connection string.
-SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%s/%s' % (POSTGRES_USER,
- POSTGRES_PASSWORD,
- POSTGRES_HOST,
- POSTGRES_PORT,
- POSTGRES_DB)
+SQLALCHEMY_DATABASE_URI = "%s://%s:%s@%s:%s/%s" % (
+ DATABASE_DIALECT,
+ DATABASE_USER,
+ DATABASE_PASSWORD,
+ DATABASE_HOST,
+ DATABASE_PORT,
+ DATABASE_DB,
+)
+
+REDIS_HOST = get_env_variable("REDIS_HOST")
+REDIS_PORT = get_env_variable("REDIS_PORT")
+REDIS_CELERY_DB = get_env_variable("REDIS_CELERY_DB", "0")
+REDIS_RESULTS_DB = get_env_variable("REDIS_RESULTS_DB", "1")
-REDIS_HOST = get_env_variable('REDIS_HOST')
-REDIS_PORT = get_env_variable('REDIS_PORT')
+RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab")
class CeleryConfig(object):
- BROKER_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
- CELERY_IMPORTS = ('superset.sql_lab', )
- CELERY_RESULT_BACKEND = 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT)
- CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}}
- CELERY_TASK_PROTOCOL = 1
+ BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
+ CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks")
+ CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}"
+ CELERYD_LOG_LEVEL = "DEBUG"
+ CELERYD_PREFETCH_MULTIPLIER = 1
+ CELERY_ACKS_LATE = False
+ CELERYBEAT_SCHEDULE = {
+ "reports.scheduler": {
+ "task": "reports.scheduler",
+ "schedule": crontab(minute="*", hour="*"),
+ },
+ "reports.prune_log": {
+ "task": "reports.prune_log",
+ "schedule": crontab(minute=10, hour=0),
+ },
+ }
CELERY_CONFIG = CeleryConfig
+FEATURE_FLAGS = {"ALERT_REPORTS": True}
+ALERT_REPORTS_NOTIFICATION_DRY_RUN = True
+WEBDRIVER_BASEURL = "http://superset:8088/"
+# The base URL for the email report hyperlinks.
+WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL
+
+SQLLAB_CTAS_NO_LIMIT = True
+
+#
+# Optionally import superset_config_docker.py (which will have been included on
+# the PYTHONPATH) in order to allow for local settings to be overridden
+#
+try:
+ import superset_config_docker
+ from superset_config_docker import * # noqa
+
+ logger.info(
+ f"Loaded your Docker configuration at " f"[{superset_config_docker.__file__}]"
+ )
+except ImportError:
+ logger.info("Using default Docker config...")
+
AUTH_TYPE = AUTH_OID
AUTH_USER_REGISTRATION = True
AUTH_USER_REGISTRATION_ROLE = "Admin"
CUSTOM_SECURITY_MANAGER = SupersetOIDCSecurityManager
-OIDC_CLIENT_SECRETS = '/home/superset/superset/id_provider.json'
+OIDC_CLIENT_SECRETS = '/home/datalab-user/superset/docker/id_provider.json'
OIDC_COOKIE_SECURE = False
OIDC_VALID_ISSUERS = 'KEYCLOAK_AUTH_SERVER_URL/realms/KEYCLOAK_REALM_NAME'
WTF_CSRF_ENABLED = False
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org