You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by be...@apache.org on 2022/01/19 19:28:59 UTC
[superset] branch master updated: chore: split CLI into multiple files (#18082)
This is an automated email from the ASF dual-hosted git repository.
beto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 9e2bc72 chore: split CLI into multiple files (#18082)
9e2bc72 is described below
commit 9e2bc72fb9687eb1d4563fd02853fea6b94a978d
Author: Beto Dealmeida <ro...@dealmeida.net>
AuthorDate: Wed Jan 19 11:27:16 2022 -0800
chore: split CLI into multiple files (#18082)
* chore: split CLI into multiple files
* Update tests
* Who fixes the fixtures?
* Add subcommands dynamically
* Rebase
---
setup.py | 2 +-
superset/cli.py | 902 -----------------------------------
superset/cli/__init__.py | 16 +
superset/cli/celery.py | 80 ++++
superset/cli/examples.py | 108 +++++
superset/cli/importexport.py | 381 +++++++++++++++
superset/cli/lib.py | 48 ++
superset/cli/main.py | 79 +++
superset/cli/test.py | 110 +++++
superset/cli/thumbnails.py | 106 ++++
superset/cli/update.py | 181 +++++++
tests/integration_tests/cli_tests.py | 104 ++--
tests/integration_tests/conftest.py | 2 +-
13 files changed, 1172 insertions(+), 947 deletions(-)
diff --git a/setup.py b/setup.py
index a2f32fd..a429a97 100644
--- a/setup.py
+++ b/setup.py
@@ -62,7 +62,7 @@ setup(
packages=find_packages(),
include_package_data=True,
zip_safe=False,
- entry_points={"console_scripts": ["superset=superset.cli:superset"]},
+ entry_points={"console_scripts": ["superset=superset.cli.main:superset"]},
install_requires=[
"backoff>=1.8.0",
"bleach>=3.0.2, <4.0.0",
diff --git a/superset/cli.py b/superset/cli.py
deleted file mode 100755
index f319316..0000000
--- a/superset/cli.py
+++ /dev/null
@@ -1,902 +0,0 @@
-#!/usr/bin/env python
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import json
-import logging
-import os
-import sys
-from datetime import datetime, timedelta
-from pathlib import Path
-from subprocess import Popen
-from typing import Any, Dict, List, Optional, Type, Union
-from zipfile import is_zipfile, ZipFile
-
-import click
-import yaml
-from apispec import APISpec
-from apispec.ext.marshmallow import MarshmallowPlugin
-from celery.utils.abstract import CallableTask
-from colorama import Fore, Style
-from flask import current_app, g
-from flask.cli import FlaskGroup, with_appcontext
-from flask_appbuilder import Model
-from flask_appbuilder.api import BaseApi
-from flask_appbuilder.api.manager import resolver
-
-import superset.utils.database as database_utils
-from superset import app, appbuilder, config, security_manager
-from superset.extensions import celery_app, db
-from superset.utils.celery import session_scope
-from superset.utils.encrypt import SecretsMigrator
-from superset.utils.urls import get_url_path
-
-logger = logging.getLogger(__name__)
-
-
-feature_flags = config.DEFAULT_FEATURE_FLAGS.copy()
-feature_flags.update(config.FEATURE_FLAGS)
-feature_flags_func = config.GET_FEATURE_FLAGS_FUNC
-if feature_flags_func:
- # pylint: disable=not-callable
- try:
- feature_flags = feature_flags_func(feature_flags)
- except Exception: # pylint: disable=broad-except
- # bypass any feature flags that depend on context
- # that's not available
- pass
-
-
-def normalize_token(token_name: str) -> str:
- """
- As of click>=7, underscores in function names are replaced by dashes.
- To avoid the need to rename all cli functions, e.g. load_examples to
- load-examples, this function is used to convert dashes back to
- underscores.
-
- :param token_name: token name possibly containing dashes
- :return: token name where dashes are replaced with underscores
- """
- return token_name.replace("_", "-")
-
-
-@click.group(
- cls=FlaskGroup, context_settings={"token_normalize_func": normalize_token},
-)
-@with_appcontext
-def superset() -> None:
- """This is a management script for the Superset application."""
-
- @app.shell_context_processor
- def make_shell_context() -> Dict[str, Any]:
- return dict(app=app, db=db)
-
-
-@superset.command()
-@with_appcontext
-def init() -> None:
- """Inits the Superset application"""
- appbuilder.add_permissions(update_perms=True)
- security_manager.sync_role_definitions()
-
-
-@superset.command()
-@with_appcontext
-@click.option("--verbose", "-v", is_flag=True, help="Show extra information")
-def version(verbose: bool) -> None:
- """Prints the current version number"""
- print(Fore.BLUE + "-=" * 15)
- print(
- Fore.YELLOW
- + "Superset "
- + Fore.CYAN
- + "{version}".format(version=app.config["VERSION_STRING"])
- )
- print(Fore.BLUE + "-=" * 15)
- if verbose:
- print("[DB] : " + "{}".format(db.engine))
- print(Style.RESET_ALL)
-
-
-def load_examples_run(
- load_test_data: bool = False,
- load_big_data: bool = False,
- only_metadata: bool = False,
- force: bool = False,
-) -> None:
- if only_metadata:
- print("Loading examples metadata")
- else:
- examples_db = database_utils.get_example_database()
- print(f"Loading examples metadata and related data into {examples_db}")
-
- # pylint: disable=import-outside-toplevel
- import superset.examples.data_loading as examples
-
- examples.load_css_templates()
-
- if load_test_data:
- print("Loading energy related dataset")
- examples.load_energy(only_metadata, force)
-
- print("Loading [World Bank's Health Nutrition and Population Stats]")
- examples.load_world_bank_health_n_pop(only_metadata, force)
-
- print("Loading [Birth names]")
- examples.load_birth_names(only_metadata, force)
-
- if load_test_data:
- print("Loading [Tabbed dashboard]")
- examples.load_tabbed_dashboard(only_metadata)
-
- if not load_test_data:
- print("Loading [Random long/lat data]")
- examples.load_long_lat_data(only_metadata, force)
-
- print("Loading [Country Map data]")
- examples.load_country_map_data(only_metadata, force)
-
- print("Loading [San Francisco population polygons]")
- examples.load_sf_population_polygons(only_metadata, force)
-
- print("Loading [Flights data]")
- examples.load_flights(only_metadata, force)
-
- print("Loading [BART lines]")
- examples.load_bart_lines(only_metadata, force)
-
- print("Loading [Multi Line]")
- examples.load_multi_line(only_metadata)
-
- print("Loading [Misc Charts] dashboard")
- examples.load_misc_dashboard()
-
- print("Loading DECK.gl demo")
- examples.load_deck_dash()
-
- if load_big_data:
- print("Loading big synthetic data for tests")
- examples.load_big_data()
-
- # load examples that are stored as YAML config files
- examples.load_examples_from_configs(force, load_test_data)
-
-
-@with_appcontext
-@superset.command()
-@click.option("--load-test-data", "-t", is_flag=True, help="Load additional test data")
-@click.option("--load-big-data", "-b", is_flag=True, help="Load additional big data")
-@click.option(
- "--only-metadata", "-m", is_flag=True, help="Only load metadata, skip actual data",
-)
-@click.option(
- "--force", "-f", is_flag=True, help="Force load data even if table already exists",
-)
-def load_examples(
- load_test_data: bool,
- load_big_data: bool,
- only_metadata: bool = False,
- force: bool = False,
-) -> None:
- """Loads a set of Slices and Dashboards and a supporting dataset"""
- load_examples_run(load_test_data, load_big_data, only_metadata, force)
-
-
-@with_appcontext
-@superset.command()
-@click.argument("directory")
-@click.option(
- "--overwrite", "-o", is_flag=True, help="Overwriting existing metadata definitions",
-)
-@click.option(
- "--force", "-f", is_flag=True, help="Force load data even if table already exists",
-)
-def import_directory(directory: str, overwrite: bool, force: bool) -> None:
- """Imports configs from a given directory"""
- # pylint: disable=import-outside-toplevel
- from superset.examples.utils import load_configs_from_directory
-
- load_configs_from_directory(
- root=Path(directory), overwrite=overwrite, force_data=force,
- )
-
-
-@with_appcontext
-@superset.command()
-@click.option("--database_name", "-d", help="Database name to change")
-@click.option("--uri", "-u", help="Database URI to change")
-@click.option(
- "--skip_create",
- "-s",
- is_flag=True,
- default=False,
- help="Create the DB if it doesn't exist",
-)
-def set_database_uri(database_name: str, uri: str, skip_create: bool) -> None:
- """Updates a database connection URI"""
- database_utils.get_or_create_db(database_name, uri, not skip_create)
-
-
-@superset.command()
-@with_appcontext
-@click.option(
- "--datasource",
- "-d",
- help="Specify which datasource name to load, if "
- "omitted, all datasources will be refreshed",
-)
-@click.option(
- "--merge",
- "-m",
- is_flag=True,
- default=False,
- help="Specify using 'merge' property during operation. " "Default value is False.",
-)
-def refresh_druid(datasource: str, merge: bool) -> None:
- """Refresh druid datasources"""
- # pylint: disable=import-outside-toplevel
- from superset.connectors.druid.models import DruidCluster
-
- session = db.session()
-
- for cluster in session.query(DruidCluster).all():
- try:
- cluster.refresh_datasources(datasource_name=datasource, merge_flag=merge)
- except Exception as ex: # pylint: disable=broad-except
- print("Error while processing cluster '{}'\n{}".format(cluster, str(ex)))
- logger.exception(ex)
- cluster.metadata_last_refreshed = datetime.now()
- print("Refreshed metadata from cluster " "[" + cluster.cluster_name + "]")
- session.commit()
-
-
-if feature_flags.get("VERSIONED_EXPORT"):
-
- @superset.command()
- @with_appcontext
- @click.option(
- "--dashboard-file", "-f", help="Specify the the file to export to",
- )
- def export_dashboards(dashboard_file: Optional[str] = None) -> None:
- """Export dashboards to ZIP file"""
- # pylint: disable=import-outside-toplevel
- from superset.dashboards.commands.export import ExportDashboardsCommand
- from superset.models.dashboard import Dashboard
-
- g.user = security_manager.find_user(username="admin")
-
- dashboard_ids = [id_ for (id_,) in db.session.query(Dashboard.id).all()]
- timestamp = datetime.now().strftime("%Y%m%dT%H%M%S")
- root = f"dashboard_export_{timestamp}"
- dashboard_file = dashboard_file or f"{root}.zip"
-
- try:
- with ZipFile(dashboard_file, "w") as bundle:
- for file_name, file_content in ExportDashboardsCommand(
- dashboard_ids
- ).run():
- with bundle.open(f"{root}/{file_name}", "w") as fp:
- fp.write(file_content.encode())
- except Exception: # pylint: disable=broad-except
- logger.exception(
- "There was an error when exporting the dashboards, please check "
- "the exception traceback in the log"
- )
- sys.exit(1)
-
- @superset.command()
- @with_appcontext
- @click.option(
- "--datasource-file", "-f", help="Specify the the file to export to",
- )
- def export_datasources(datasource_file: Optional[str] = None) -> None:
- """Export datasources to ZIP file"""
- # pylint: disable=import-outside-toplevel
- from superset.connectors.sqla.models import SqlaTable
- from superset.datasets.commands.export import ExportDatasetsCommand
-
- g.user = security_manager.find_user(username="admin")
-
- dataset_ids = [id_ for (id_,) in db.session.query(SqlaTable.id).all()]
- timestamp = datetime.now().strftime("%Y%m%dT%H%M%S")
- root = f"dataset_export_{timestamp}"
- datasource_file = datasource_file or f"{root}.zip"
-
- try:
- with ZipFile(datasource_file, "w") as bundle:
- for file_name, file_content in ExportDatasetsCommand(dataset_ids).run():
- with bundle.open(f"{root}/{file_name}", "w") as fp:
- fp.write(file_content.encode())
- except Exception: # pylint: disable=broad-except
- logger.exception(
- "There was an error when exporting the datasets, please check "
- "the exception traceback in the log"
- )
- sys.exit(1)
-
- @superset.command()
- @with_appcontext
- @click.option(
- "--path", "-p", help="Path to a single ZIP file",
- )
- @click.option(
- "--username",
- "-u",
- default=None,
- help="Specify the user name to assign dashboards to",
- )
- def import_dashboards(path: str, username: Optional[str]) -> None:
- """Import dashboards from ZIP file"""
- # pylint: disable=import-outside-toplevel
- from superset.commands.importers.v1.utils import get_contents_from_bundle
- from superset.dashboards.commands.importers.dispatcher import (
- ImportDashboardsCommand,
- )
-
- if username is not None:
- g.user = security_manager.find_user(username=username)
- if is_zipfile(path):
- with ZipFile(path) as bundle:
- contents = get_contents_from_bundle(bundle)
- else:
- with open(path) as file:
- contents = {path: file.read()}
- try:
- ImportDashboardsCommand(contents, overwrite=True).run()
- except Exception: # pylint: disable=broad-except
- logger.exception(
- "There was an error when importing the dashboards(s), please check "
- "the exception traceback in the log"
- )
- sys.exit(1)
-
- @superset.command()
- @with_appcontext
- @click.option(
- "--path", "-p", help="Path to a single ZIP file",
- )
- def import_datasources(path: str) -> None:
- """Import datasources from ZIP file"""
- # pylint: disable=import-outside-toplevel
- from superset.commands.importers.v1.utils import get_contents_from_bundle
- from superset.datasets.commands.importers.dispatcher import (
- ImportDatasetsCommand,
- )
-
- if is_zipfile(path):
- with ZipFile(path) as bundle:
- contents = get_contents_from_bundle(bundle)
- else:
- with open(path) as file:
- contents = {path: file.read()}
- try:
- ImportDatasetsCommand(contents, overwrite=True).run()
- except Exception: # pylint: disable=broad-except
- logger.exception(
- "There was an error when importing the dataset(s), please check the "
- "exception traceback in the log"
- )
- sys.exit(1)
-
-
-else:
-
- @superset.command()
- @with_appcontext
- @click.option(
- "--dashboard-file",
- "-f",
- default=None,
- help="Specify the the file to export to",
- )
- @click.option(
- "--print_stdout",
- "-p",
- is_flag=True,
- default=False,
- help="Print JSON to stdout",
- )
- def export_dashboards(
- dashboard_file: Optional[str], print_stdout: bool = False
- ) -> None:
- """Export dashboards to JSON"""
- # pylint: disable=import-outside-toplevel
- from superset.utils import dashboard_import_export
-
- data = dashboard_import_export.export_dashboards(db.session)
- if print_stdout or not dashboard_file:
- print(data)
- if dashboard_file:
- logger.info("Exporting dashboards to %s", dashboard_file)
- with open(dashboard_file, "w") as data_stream:
- data_stream.write(data)
-
- @superset.command()
- @with_appcontext
- @click.option(
- "--datasource-file",
- "-f",
- default=None,
- help="Specify the the file to export to",
- )
- @click.option(
- "--print_stdout",
- "-p",
- is_flag=True,
- default=False,
- help="Print YAML to stdout",
- )
- @click.option(
- "--back-references",
- "-b",
- is_flag=True,
- default=False,
- help="Include parent back references",
- )
- @click.option(
- "--include-defaults",
- "-d",
- is_flag=True,
- default=False,
- help="Include fields containing defaults",
- )
- def export_datasources(
- datasource_file: Optional[str],
- print_stdout: bool = False,
- back_references: bool = False,
- include_defaults: bool = False,
- ) -> None:
- """Export datasources to YAML"""
- # pylint: disable=import-outside-toplevel
- from superset.utils import dict_import_export
-
- data = dict_import_export.export_to_dict(
- session=db.session,
- recursive=True,
- back_references=back_references,
- include_defaults=include_defaults,
- )
- if print_stdout or not datasource_file:
- yaml.safe_dump(data, sys.stdout, default_flow_style=False)
- if datasource_file:
- logger.info("Exporting datasources to %s", datasource_file)
- with open(datasource_file, "w") as data_stream:
- yaml.safe_dump(data, data_stream, default_flow_style=False)
-
- @superset.command()
- @with_appcontext
- @click.option(
- "--path",
- "-p",
- help="Path to a single JSON file or path containing multiple JSON "
- "files to import (*.json)",
- )
- @click.option(
- "--recursive",
- "-r",
- is_flag=True,
- default=False,
- help="recursively search the path for json files",
- )
- @click.option(
- "--username",
- "-u",
- default=None,
- help="Specify the user name to assign dashboards to",
- )
- def import_dashboards(path: str, recursive: bool, username: str) -> None:
- """Import dashboards from JSON file"""
- # pylint: disable=import-outside-toplevel
- from superset.dashboards.commands.importers.v0 import ImportDashboardsCommand
-
- path_object = Path(path)
- files: List[Path] = []
- if path_object.is_file():
- files.append(path_object)
- elif path_object.exists() and not recursive:
- files.extend(path_object.glob("*.json"))
- elif path_object.exists() and recursive:
- files.extend(path_object.rglob("*.json"))
- if username is not None:
- g.user = security_manager.find_user(username=username)
- contents = {}
- for path_ in files:
- with open(path_) as file:
- contents[path_.name] = file.read()
- try:
- ImportDashboardsCommand(contents).run()
- except Exception: # pylint: disable=broad-except
- logger.exception("Error when importing dashboard")
- sys.exit(1)
-
- @superset.command()
- @with_appcontext
- @click.option(
- "--path",
- "-p",
- help="Path to a single YAML file or path containing multiple YAML "
- "files to import (*.yaml or *.yml)",
- )
- @click.option(
- "--sync",
- "-s",
- "sync",
- default="",
- help="comma seperated list of element types to synchronize "
- 'e.g. "metrics,columns" deletes metrics and columns in the DB '
- "that are not specified in the YAML file",
- )
- @click.option(
- "--recursive",
- "-r",
- is_flag=True,
- default=False,
- help="recursively search the path for yaml files",
- )
- def import_datasources(path: str, sync: str, recursive: bool) -> None:
- """Import datasources from YAML"""
- # pylint: disable=import-outside-toplevel
- from superset.datasets.commands.importers.v0 import ImportDatasetsCommand
-
- sync_array = sync.split(",")
- sync_columns = "columns" in sync_array
- sync_metrics = "metrics" in sync_array
-
- path_object = Path(path)
- files: List[Path] = []
- if path_object.is_file():
- files.append(path_object)
- elif path_object.exists() and not recursive:
- files.extend(path_object.glob("*.yaml"))
- files.extend(path_object.glob("*.yml"))
- elif path_object.exists() and recursive:
- files.extend(path_object.rglob("*.yaml"))
- files.extend(path_object.rglob("*.yml"))
- contents = {}
- for path_ in files:
- with open(path_) as file:
- contents[path_.name] = file.read()
- try:
- ImportDatasetsCommand(contents, sync_columns, sync_metrics).run()
- except Exception: # pylint: disable=broad-except
- logger.exception("Error when importing dataset")
- sys.exit(1)
-
- @superset.command()
- @with_appcontext
- @click.option(
- "--back-references",
- "-b",
- is_flag=True,
- default=False,
- help="Include parent back references",
- )
- def export_datasource_schema(back_references: bool) -> None:
- """Export datasource YAML schema to stdout"""
- # pylint: disable=import-outside-toplevel
- from superset.utils import dict_import_export
-
- data = dict_import_export.export_schema_to_dict(back_references=back_references)
- yaml.safe_dump(data, sys.stdout, default_flow_style=False)
-
-
-@superset.command()
-@with_appcontext
-def update_datasources_cache() -> None:
- """Refresh sqllab datasources cache"""
- # pylint: disable=import-outside-toplevel
- from superset.models.core import Database
-
- for database in db.session.query(Database).all():
- if database.allow_multi_schema_metadata_fetch:
- print("Fetching {} datasources ...".format(database.name))
- try:
- database.get_all_table_names_in_database(
- force=True, cache=True, cache_timeout=24 * 60 * 60
- )
- database.get_all_view_names_in_database(
- force=True, cache=True, cache_timeout=24 * 60 * 60
- )
- except Exception as ex: # pylint: disable=broad-except
- print("{}".format(str(ex)))
-
-
-@superset.command()
-@with_appcontext
-@click.option(
- "--workers", "-w", type=int, help="Number of celery server workers to fire up",
-)
-def worker(workers: int) -> None:
- """Starts a Superset worker for async SQL query execution."""
- logger.info(
- "The 'superset worker' command is deprecated. Please use the 'celery "
- "worker' command instead."
- )
- if workers:
- celery_app.conf.update(CELERYD_CONCURRENCY=workers)
- elif app.config["SUPERSET_CELERY_WORKERS"]:
- celery_app.conf.update(
- CELERYD_CONCURRENCY=app.config["SUPERSET_CELERY_WORKERS"]
- )
-
- local_worker = celery_app.Worker(optimization="fair")
- local_worker.start()
-
-
-@superset.command()
-@with_appcontext
-@click.option(
- "-p", "--port", default="5555", help="Port on which to start the Flower process",
-)
-@click.option(
- "-a", "--address", default="localhost", help="Address on which to run the service",
-)
-def flower(port: int, address: str) -> None:
- """Runs a Celery Flower web server
-
- Celery Flower is a UI to monitor the Celery operation on a given
- broker"""
- broker_url = celery_app.conf.BROKER_URL
- cmd = (
- "celery flower "
- f"--broker={broker_url} "
- f"--port={port} "
- f"--address={address} "
- )
- logger.info(
- "The 'superset flower' command is deprecated. Please use the 'celery "
- "flower' command instead."
- )
- print(Fore.GREEN + "Starting a Celery Flower instance")
- print(Fore.BLUE + "-=" * 40)
- print(Fore.YELLOW + cmd)
- print(Fore.BLUE + "-=" * 40)
- Popen(cmd, shell=True).wait() # pylint: disable=consider-using-with
-
-
-@superset.command()
-@with_appcontext
-@click.option(
- "--asynchronous",
- "-a",
- is_flag=True,
- default=False,
- help="Trigger commands to run remotely on a worker",
-)
-@click.option(
- "--dashboards_only",
- "-d",
- is_flag=True,
- default=False,
- help="Only process dashboards",
-)
-@click.option(
- "--charts_only", "-c", is_flag=True, default=False, help="Only process charts",
-)
-@click.option(
- "--force",
- "-f",
- is_flag=True,
- default=False,
- help="Force refresh, even if previously cached",
-)
-@click.option("--model_id", "-i", multiple=True)
-def compute_thumbnails(
- asynchronous: bool,
- dashboards_only: bool,
- charts_only: bool,
- force: bool,
- model_id: int,
-) -> None:
- """Compute thumbnails"""
- # pylint: disable=import-outside-toplevel
- from superset.models.dashboard import Dashboard
- from superset.models.slice import Slice
- from superset.tasks.thumbnails import (
- cache_chart_thumbnail,
- cache_dashboard_thumbnail,
- )
-
- def compute_generic_thumbnail(
- friendly_type: str,
- model_cls: Union[Type[Dashboard], Type[Slice]],
- model_id: int,
- compute_func: CallableTask,
- ) -> None:
- query = db.session.query(model_cls)
- if model_id:
- query = query.filter(model_cls.id.in_(model_id))
- dashboards = query.all()
- count = len(dashboards)
- for i, model in enumerate(dashboards):
- if asynchronous:
- func = compute_func.delay
- action = "Triggering"
- else:
- func = compute_func
- action = "Processing"
- msg = f'{action} {friendly_type} "{model}" ({i+1}/{count})'
- click.secho(msg, fg="green")
- if friendly_type == "chart":
- url = get_url_path(
- "Superset.slice", slice_id=model.id, standalone="true"
- )
- else:
- url = get_url_path("Superset.dashboard", dashboard_id_or_slug=model.id)
- func(url, model.digest, force=force)
-
- if not charts_only:
- compute_generic_thumbnail(
- "dashboard", Dashboard, model_id, cache_dashboard_thumbnail
- )
- if not dashboards_only:
- compute_generic_thumbnail("chart", Slice, model_id, cache_chart_thumbnail)
-
-
-@superset.command()
-@with_appcontext
-def load_test_users() -> None:
- """
- Loads admin, alpha, and gamma user for testing purposes
-
- Syncs permissions for those users/roles
- """
- print(Fore.GREEN + "Loading a set of users for unit tests")
- load_test_users_run()
-
-
-def load_test_users_run() -> None:
- """
- Loads admin, alpha, and gamma user for testing purposes
-
- Syncs permissions for those users/roles
- """
- if app.config["TESTING"]:
-
- sm = security_manager
-
- examples_db = database_utils.get_example_database()
-
- examples_pv = sm.add_permission_view_menu("database_access", examples_db.perm)
-
- sm.sync_role_definitions()
- gamma_sqllab_role = sm.add_role("gamma_sqllab")
- sm.add_permission_role(gamma_sqllab_role, examples_pv)
-
- gamma_no_csv_role = sm.add_role("gamma_no_csv")
- sm.add_permission_role(gamma_no_csv_role, examples_pv)
-
- for role in ["Gamma", "sql_lab"]:
- for perm in sm.find_role(role).permissions:
- sm.add_permission_role(gamma_sqllab_role, perm)
-
- if str(perm) != "can csv on Superset":
- sm.add_permission_role(gamma_no_csv_role, perm)
-
- users = (
- ("admin", "Admin"),
- ("gamma", "Gamma"),
- ("gamma2", "Gamma"),
- ("gamma_sqllab", "gamma_sqllab"),
- ("alpha", "Alpha"),
- ("gamma_no_csv", "gamma_no_csv"),
- )
- for username, role in users:
- user = sm.find_user(username)
- if not user:
- sm.add_user(
- username,
- username,
- "user",
- username + "@fab.org",
- sm.find_role(role),
- password="general",
- )
- sm.get_session.commit()
-
-
-@superset.command()
-@with_appcontext
-def sync_tags() -> None:
- """Rebuilds special tags (owner, type, favorited by)."""
- # pylint: disable=no-member
- metadata = Model.metadata
-
- # pylint: disable=import-outside-toplevel
- from superset.common.tags import add_favorites, add_owners, add_types
-
- add_types(db.engine, metadata)
- add_owners(db.engine, metadata)
- add_favorites(db.engine, metadata)
-
-
-@superset.command()
-@with_appcontext
-def alert() -> None:
- """Run the alert scheduler loop"""
- # this command is just for testing purposes
- # pylint: disable=import-outside-toplevel
- from superset.models.schedules import ScheduleType
- from superset.tasks.schedules import schedule_window
-
- click.secho("Processing one alert loop", fg="green")
- with session_scope(nullpool=True) as session:
- schedule_window(
- report_type=ScheduleType.alert,
- start_at=datetime.now() - timedelta(1000),
- stop_at=datetime.now(),
- resolution=6000,
- session=session,
- )
-
-
-@superset.command()
-@with_appcontext
-def update_api_docs() -> None:
- """Regenerate the openapi.json file in docs"""
- superset_dir = os.path.abspath(os.path.dirname(__file__))
- openapi_json = os.path.join(
- superset_dir, "..", "docs", "src", "resources", "openapi.json"
- )
- api_version = "v1"
-
- version_found = False
- api_spec = APISpec(
- title=current_app.appbuilder.app_name,
- version=api_version,
- openapi_version="3.0.2",
- info=dict(description=current_app.appbuilder.app_name),
- plugins=[MarshmallowPlugin(schema_name_resolver=resolver)],
- servers=[{"url": "http://localhost:8088"}],
- )
- for base_api in current_app.appbuilder.baseviews:
- if isinstance(base_api, BaseApi) and base_api.version == api_version:
- base_api.add_api_spec(api_spec)
- version_found = True
- if version_found:
- click.secho("Generating openapi.json", fg="green")
- with open(openapi_json, "w") as outfile:
- json.dump(api_spec.to_dict(), outfile, sort_keys=True, indent=2)
- else:
- click.secho("API version not found", err=True)
-
-
-@superset.command()
-@with_appcontext
-@click.option(
- "--previous_secret_key",
- "-a",
- required=False,
- help="An optional previous secret key, if PREVIOUS_SECRET_KEY "
- "is not set on the config",
-)
-def re_encrypt_secrets(previous_secret_key: Optional[str] = None) -> None:
- previous_secret_key = previous_secret_key or current_app.config.get(
- "PREVIOUS_SECRET_KEY"
- )
- if previous_secret_key is None:
- click.secho("A previous secret key must be provided", err=True)
- sys.exit(1)
- secrets_migrator = SecretsMigrator(previous_secret_key=previous_secret_key)
- try:
- secrets_migrator.run()
- except ValueError as exc:
- click.secho(
- f"An error occurred, "
- f"probably an invalid previoud secret key was provided. Error:[{exc}]",
- err=True,
- )
- sys.exit(1)
diff --git a/superset/cli/__init__.py b/superset/cli/__init__.py
new file mode 100644
index 0000000..13a8339
--- /dev/null
+++ b/superset/cli/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/superset/cli/celery.py b/superset/cli/celery.py
new file mode 100755
index 0000000..a037357
--- /dev/null
+++ b/superset/cli/celery.py
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import logging
+from subprocess import Popen
+
+import click
+from colorama import Fore
+from flask.cli import with_appcontext
+
+from superset import app
+from superset.extensions import celery_app
+
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@with_appcontext
+@click.option(
+ "--workers", "-w", type=int, help="Number of celery server workers to fire up",
+)
+def worker(workers: int) -> None:
+ """Starts a Superset worker for async SQL query execution."""
+ logger.info(
+ "The 'superset worker' command is deprecated. Please use the 'celery "
+ "worker' command instead."
+ )
+ if workers:
+ celery_app.conf.update(CELERYD_CONCURRENCY=workers)
+ elif app.config["SUPERSET_CELERY_WORKERS"]:
+ celery_app.conf.update(
+ CELERYD_CONCURRENCY=app.config["SUPERSET_CELERY_WORKERS"]
+ )
+
+ local_worker = celery_app.Worker(optimization="fair")
+ local_worker.start()
+
+
+@click.command()
+@with_appcontext
+@click.option(
+ "-p", "--port", default="5555", help="Port on which to start the Flower process",
+)
+@click.option(
+ "-a", "--address", default="localhost", help="Address on which to run the service",
+)
+def flower(port: int, address: str) -> None:
+ """Runs a Celery Flower web server
+
+ Celery Flower is a UI to monitor the Celery operation on a given
+ broker"""
+ broker_url = celery_app.conf.BROKER_URL
+ cmd = (
+ "celery flower "
+ f"--broker={broker_url} "
+ f"--port={port} "
+ f"--address={address} "
+ )
+ logger.info(
+ "The 'superset flower' command is deprecated. Please use the 'celery "
+ "flower' command instead."
+ )
+ print(Fore.GREEN + "Starting a Celery Flower instance")
+ print(Fore.BLUE + "-=" * 40)
+ print(Fore.YELLOW + cmd)
+ print(Fore.BLUE + "-=" * 40)
+ Popen(cmd, shell=True).wait() # pylint: disable=consider-using-with
diff --git a/superset/cli/examples.py b/superset/cli/examples.py
new file mode 100755
index 0000000..9739463
--- /dev/null
+++ b/superset/cli/examples.py
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import logging
+
+import click
+from flask.cli import with_appcontext
+
+import superset.utils.database as database_utils
+
+logger = logging.getLogger(__name__)
+
+
+def load_examples_run(
+ load_test_data: bool = False,
+ load_big_data: bool = False,
+ only_metadata: bool = False,
+ force: bool = False,
+) -> None:
+ if only_metadata:
+ print("Loading examples metadata")
+ else:
+ examples_db = database_utils.get_example_database()
+ print(f"Loading examples metadata and related data into {examples_db}")
+
+ # pylint: disable=import-outside-toplevel
+ import superset.examples.data_loading as examples
+
+ examples.load_css_templates()
+
+ if load_test_data:
+ print("Loading energy related dataset")
+ examples.load_energy(only_metadata, force)
+
+ print("Loading [World Bank's Health Nutrition and Population Stats]")
+ examples.load_world_bank_health_n_pop(only_metadata, force)
+
+ print("Loading [Birth names]")
+ examples.load_birth_names(only_metadata, force)
+
+ if load_test_data:
+ print("Loading [Tabbed dashboard]")
+ examples.load_tabbed_dashboard(only_metadata)
+
+ if not load_test_data:
+ print("Loading [Random long/lat data]")
+ examples.load_long_lat_data(only_metadata, force)
+
+ print("Loading [Country Map data]")
+ examples.load_country_map_data(only_metadata, force)
+
+ print("Loading [San Francisco population polygons]")
+ examples.load_sf_population_polygons(only_metadata, force)
+
+ print("Loading [Flights data]")
+ examples.load_flights(only_metadata, force)
+
+ print("Loading [BART lines]")
+ examples.load_bart_lines(only_metadata, force)
+
+ print("Loading [Multi Line]")
+ examples.load_multi_line(only_metadata)
+
+ print("Loading [Misc Charts] dashboard")
+ examples.load_misc_dashboard()
+
+ print("Loading DECK.gl demo")
+ examples.load_deck_dash()
+
+ if load_big_data:
+ print("Loading big synthetic data for tests")
+ examples.load_big_data()
+
+ # load examples that are stored as YAML config files
+ examples.load_examples_from_configs(force, load_test_data)
+
+
+@click.command()
+@with_appcontext
+@click.option("--load-test-data", "-t", is_flag=True, help="Load additional test data")
+@click.option("--load-big-data", "-b", is_flag=True, help="Load additional big data")
+@click.option(
+ "--only-metadata", "-m", is_flag=True, help="Only load metadata, skip actual data",
+)
+@click.option(
+ "--force", "-f", is_flag=True, help="Force load data even if table already exists",
+)
+def load_examples(
+ load_test_data: bool,
+ load_big_data: bool,
+ only_metadata: bool = False,
+ force: bool = False,
+) -> None:
+ """Loads a set of Slices and Dashboards and a supporting dataset"""
+ load_examples_run(load_test_data, load_big_data, only_metadata, force)
diff --git a/superset/cli/importexport.py b/superset/cli/importexport.py
new file mode 100755
index 0000000..21167bc
--- /dev/null
+++ b/superset/cli/importexport.py
@@ -0,0 +1,381 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import logging
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+from zipfile import is_zipfile, ZipFile
+
+import click
+import yaml
+from flask import g
+from flask.cli import with_appcontext
+
+from superset import security_manager
+from superset.cli.lib import feature_flags
+from superset.extensions import db
+
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.argument("directory")
+@click.option(
+ "--overwrite", "-o", is_flag=True, help="Overwriting existing metadata definitions",
+)
+@click.option(
+ "--force", "-f", is_flag=True, help="Force load data even if table already exists",
+)
+def import_directory(directory: str, overwrite: bool, force: bool) -> None:
+ """Imports configs from a given directory"""
+ # pylint: disable=import-outside-toplevel
+ from superset.examples.utils import load_configs_from_directory
+
+ load_configs_from_directory(
+ root=Path(directory), overwrite=overwrite, force_data=force,
+ )
+
+
+if feature_flags.get("VERSIONED_EXPORT"):
+
+ @click.command()
+ @with_appcontext
+ @click.option(
+ "--dashboard-file", "-f", help="Specify the the file to export to",
+ )
+ def export_dashboards(dashboard_file: Optional[str] = None) -> None:
+ """Export dashboards to ZIP file"""
+ # pylint: disable=import-outside-toplevel
+ from superset.dashboards.commands.export import ExportDashboardsCommand
+ from superset.models.dashboard import Dashboard
+
+ g.user = security_manager.find_user(username="admin")
+
+ dashboard_ids = [id_ for (id_,) in db.session.query(Dashboard.id).all()]
+ timestamp = datetime.now().strftime("%Y%m%dT%H%M%S")
+ root = f"dashboard_export_{timestamp}"
+ dashboard_file = dashboard_file or f"{root}.zip"
+
+ try:
+ with ZipFile(dashboard_file, "w") as bundle:
+ for file_name, file_content in ExportDashboardsCommand(
+ dashboard_ids
+ ).run():
+ with bundle.open(f"{root}/{file_name}", "w") as fp:
+ fp.write(file_content.encode())
+ except Exception: # pylint: disable=broad-except
+ logger.exception(
+ "There was an error when exporting the dashboards, please check "
+ "the exception traceback in the log"
+ )
+ sys.exit(1)
+
+ @click.command()
+ @with_appcontext
+ @click.option(
+ "--datasource-file", "-f", help="Specify the the file to export to",
+ )
+ def export_datasources(datasource_file: Optional[str] = None) -> None:
+ """Export datasources to ZIP file"""
+ # pylint: disable=import-outside-toplevel
+ from superset.connectors.sqla.models import SqlaTable
+ from superset.datasets.commands.export import ExportDatasetsCommand
+
+ g.user = security_manager.find_user(username="admin")
+
+ dataset_ids = [id_ for (id_,) in db.session.query(SqlaTable.id).all()]
+ timestamp = datetime.now().strftime("%Y%m%dT%H%M%S")
+ root = f"dataset_export_{timestamp}"
+ datasource_file = datasource_file or f"{root}.zip"
+
+ try:
+ with ZipFile(datasource_file, "w") as bundle:
+ for file_name, file_content in ExportDatasetsCommand(dataset_ids).run():
+ with bundle.open(f"{root}/{file_name}", "w") as fp:
+ fp.write(file_content.encode())
+ except Exception: # pylint: disable=broad-except
+ logger.exception(
+ "There was an error when exporting the datasets, please check "
+ "the exception traceback in the log"
+ )
+ sys.exit(1)
+
+ @click.command()
+ @with_appcontext
+ @click.option(
+ "--path", "-p", help="Path to a single ZIP file",
+ )
+ @click.option(
+ "--username",
+ "-u",
+ default=None,
+ help="Specify the user name to assign dashboards to",
+ )
+ def import_dashboards(path: str, username: Optional[str]) -> None:
+ """Import dashboards from ZIP file"""
+ # pylint: disable=import-outside-toplevel
+ from superset.commands.importers.v1.utils import get_contents_from_bundle
+ from superset.dashboards.commands.importers.dispatcher import (
+ ImportDashboardsCommand,
+ )
+
+ if username is not None:
+ g.user = security_manager.find_user(username=username)
+ if is_zipfile(path):
+ with ZipFile(path) as bundle:
+ contents = get_contents_from_bundle(bundle)
+ else:
+ with open(path) as file:
+ contents = {path: file.read()}
+ try:
+ ImportDashboardsCommand(contents, overwrite=True).run()
+ except Exception: # pylint: disable=broad-except
+ logger.exception(
+ "There was an error when importing the dashboards(s), please check "
+ "the exception traceback in the log"
+ )
+ sys.exit(1)
+
+ @click.command()
+ @with_appcontext
+ @click.option(
+ "--path", "-p", help="Path to a single ZIP file",
+ )
+ def import_datasources(path: str) -> None:
+ """Import datasources from ZIP file"""
+ # pylint: disable=import-outside-toplevel
+ from superset.commands.importers.v1.utils import get_contents_from_bundle
+ from superset.datasets.commands.importers.dispatcher import (
+ ImportDatasetsCommand,
+ )
+
+ if is_zipfile(path):
+ with ZipFile(path) as bundle:
+ contents = get_contents_from_bundle(bundle)
+ else:
+ with open(path) as file:
+ contents = {path: file.read()}
+ try:
+ ImportDatasetsCommand(contents, overwrite=True).run()
+ except Exception: # pylint: disable=broad-except
+ logger.exception(
+ "There was an error when importing the dataset(s), please check the "
+ "exception traceback in the log"
+ )
+ sys.exit(1)
+
+
+else:
+
+ @click.command()
+ @with_appcontext
+ @click.option(
+ "--dashboard-file",
+ "-f",
+ default=None,
+ help="Specify the the file to export to",
+ )
+ @click.option(
+ "--print_stdout",
+ "-p",
+ is_flag=True,
+ default=False,
+ help="Print JSON to stdout",
+ )
+ def export_dashboards(
+ dashboard_file: Optional[str], print_stdout: bool = False
+ ) -> None:
+ """Export dashboards to JSON"""
+ # pylint: disable=import-outside-toplevel
+ from superset.utils import dashboard_import_export
+
+ data = dashboard_import_export.export_dashboards(db.session)
+ if print_stdout or not dashboard_file:
+ print(data)
+ if dashboard_file:
+ logger.info("Exporting dashboards to %s", dashboard_file)
+ with open(dashboard_file, "w") as data_stream:
+ data_stream.write(data)
+
+ @click.command()
+ @with_appcontext
+ @click.option(
+ "--datasource-file",
+ "-f",
+ default=None,
+ help="Specify the the file to export to",
+ )
+ @click.option(
+ "--print_stdout",
+ "-p",
+ is_flag=True,
+ default=False,
+ help="Print YAML to stdout",
+ )
+ @click.option(
+ "--back-references",
+ "-b",
+ is_flag=True,
+ default=False,
+ help="Include parent back references",
+ )
+ @click.option(
+ "--include-defaults",
+ "-d",
+ is_flag=True,
+ default=False,
+ help="Include fields containing defaults",
+ )
+ def export_datasources(
+ datasource_file: Optional[str],
+ print_stdout: bool = False,
+ back_references: bool = False,
+ include_defaults: bool = False,
+ ) -> None:
+ """Export datasources to YAML"""
+ # pylint: disable=import-outside-toplevel
+ from superset.utils import dict_import_export
+
+ data = dict_import_export.export_to_dict(
+ session=db.session,
+ recursive=True,
+ back_references=back_references,
+ include_defaults=include_defaults,
+ )
+ if print_stdout or not datasource_file:
+ yaml.safe_dump(data, sys.stdout, default_flow_style=False)
+ if datasource_file:
+ logger.info("Exporting datasources to %s", datasource_file)
+ with open(datasource_file, "w") as data_stream:
+ yaml.safe_dump(data, data_stream, default_flow_style=False)
+
+ @click.command()
+ @with_appcontext
+ @click.option(
+ "--path",
+ "-p",
+ help="Path to a single JSON file or path containing multiple JSON "
+ "files to import (*.json)",
+ )
+ @click.option(
+ "--recursive",
+ "-r",
+ is_flag=True,
+ default=False,
+ help="recursively search the path for json files",
+ )
+ @click.option(
+ "--username",
+ "-u",
+ default=None,
+ help="Specify the user name to assign dashboards to",
+ )
+ def import_dashboards(path: str, recursive: bool, username: str) -> None:
+ """Import dashboards from JSON file"""
+ # pylint: disable=import-outside-toplevel
+ from superset.dashboards.commands.importers.v0 import ImportDashboardsCommand
+
+ path_object = Path(path)
+ files: List[Path] = []
+ if path_object.is_file():
+ files.append(path_object)
+ elif path_object.exists() and not recursive:
+ files.extend(path_object.glob("*.json"))
+ elif path_object.exists() and recursive:
+ files.extend(path_object.rglob("*.json"))
+ if username is not None:
+ g.user = security_manager.find_user(username=username)
+ contents = {}
+ for path_ in files:
+ with open(path_) as file:
+ contents[path_.name] = file.read()
+ try:
+ ImportDashboardsCommand(contents).run()
+ except Exception: # pylint: disable=broad-except
+ logger.exception("Error when importing dashboard")
+ sys.exit(1)
+
+ @click.command()
+ @with_appcontext
+ @click.option(
+ "--path",
+ "-p",
+ help="Path to a single YAML file or path containing multiple YAML "
+ "files to import (*.yaml or *.yml)",
+ )
+ @click.option(
+ "--sync",
+ "-s",
+ "sync",
+ default="",
+ help="comma seperated list of element types to synchronize "
+ 'e.g. "metrics,columns" deletes metrics and columns in the DB '
+ "that are not specified in the YAML file",
+ )
+ @click.option(
+ "--recursive",
+ "-r",
+ is_flag=True,
+ default=False,
+ help="recursively search the path for yaml files",
+ )
+ def import_datasources(path: str, sync: str, recursive: bool) -> None:
+ """Import datasources from YAML"""
+ # pylint: disable=import-outside-toplevel
+ from superset.datasets.commands.importers.v0 import ImportDatasetsCommand
+
+ sync_array = sync.split(",")
+ sync_columns = "columns" in sync_array
+ sync_metrics = "metrics" in sync_array
+
+ path_object = Path(path)
+ files: List[Path] = []
+ if path_object.is_file():
+ files.append(path_object)
+ elif path_object.exists() and not recursive:
+ files.extend(path_object.glob("*.yaml"))
+ files.extend(path_object.glob("*.yml"))
+ elif path_object.exists() and recursive:
+ files.extend(path_object.rglob("*.yaml"))
+ files.extend(path_object.rglob("*.yml"))
+ contents = {}
+ for path_ in files:
+ with open(path_) as file:
+ contents[path_.name] = file.read()
+ try:
+ ImportDatasetsCommand(contents, sync_columns, sync_metrics).run()
+ except Exception: # pylint: disable=broad-except
+ logger.exception("Error when importing dataset")
+ sys.exit(1)
+
+ @click.command()
+ @with_appcontext
+ @click.option(
+ "--back-references",
+ "-b",
+ is_flag=True,
+ default=False,
+ help="Include parent back references",
+ )
+ def export_datasource_schema(back_references: bool) -> None:
+ """Export datasource YAML schema to stdout"""
+ # pylint: disable=import-outside-toplevel
+ from superset.utils import dict_import_export
+
+ data = dict_import_export.export_schema_to_dict(back_references=back_references)
+ yaml.safe_dump(data, sys.stdout, default_flow_style=False)
diff --git a/superset/cli/lib.py b/superset/cli/lib.py
new file mode 100755
index 0000000..9e14ab6
--- /dev/null
+++ b/superset/cli/lib.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import logging
+
+from superset import config
+
+logger = logging.getLogger(__name__)
+
+
+feature_flags = config.DEFAULT_FEATURE_FLAGS.copy()
+feature_flags.update(config.FEATURE_FLAGS)
+feature_flags_func = config.GET_FEATURE_FLAGS_FUNC
+if feature_flags_func:
+ # pylint: disable=not-callable
+ try:
+ feature_flags = feature_flags_func(feature_flags)
+ except Exception: # pylint: disable=broad-except
+ # bypass any feature flags that depend on context
+ # that's not available
+ pass
+
+
+def normalize_token(token_name: str) -> str:
+ """
+ As of click>=7, underscores in function names are replaced by dashes.
+ To avoid the need to rename all cli functions, e.g. load_examples to
+ load-examples, this function is used to convert dashes back to
+ underscores.
+
+ :param token_name: token name possibly containing dashes
+ :return: token name where dashes are replaced with underscores
+ """
+ return token_name.replace("_", "-")
diff --git a/superset/cli/main.py b/superset/cli/main.py
new file mode 100755
index 0000000..45b4c9e
--- /dev/null
+++ b/superset/cli/main.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import importlib
+import logging
+import pkgutil
+from typing import Any, Dict
+
+import click
+from colorama import Fore, Style
+from flask.cli import FlaskGroup, with_appcontext
+
+from superset import app, appbuilder, cli, security_manager
+from superset.cli.lib import normalize_token
+from superset.extensions import db
+
+logger = logging.getLogger(__name__)
+
+
+@click.group(
+ cls=FlaskGroup, context_settings={"token_normalize_func": normalize_token},
+)
+@with_appcontext
+def superset() -> None:
+ """This is a management script for the Superset application."""
+
+ @app.shell_context_processor
+ def make_shell_context() -> Dict[str, Any]:
+ return dict(app=app, db=db)
+
+
+# add sub-commands
+for load, module_name, is_pkg in pkgutil.walk_packages(
+ cli.__path__, cli.__name__ + "." # type: ignore
+):
+ module = importlib.import_module(module_name)
+ for attribute in module.__dict__.values():
+ if isinstance(attribute, click.core.Command):
+ superset.add_command(attribute)
+
+
+@superset.command()
+@with_appcontext
+def init() -> None:
+ """Inits the Superset application"""
+ appbuilder.add_permissions(update_perms=True)
+ security_manager.sync_role_definitions()
+
+
+@superset.command()
+@with_appcontext
+@click.option("--verbose", "-v", is_flag=True, help="Show extra information")
+def version(verbose: bool) -> None:
+ """Prints the current version number"""
+ print(Fore.BLUE + "-=" * 15)
+ print(
+ Fore.YELLOW
+ + "Superset "
+ + Fore.CYAN
+ + "{version}".format(version=app.config["VERSION_STRING"])
+ )
+ print(Fore.BLUE + "-=" * 15)
+ if verbose:
+ print("[DB] : " + "{}".format(db.engine))
+ print(Style.RESET_ALL)
diff --git a/superset/cli/test.py b/superset/cli/test.py
new file mode 100755
index 0000000..df0142b
--- /dev/null
+++ b/superset/cli/test.py
@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import logging
+from datetime import datetime, timedelta
+
+import click
+from colorama import Fore
+from flask.cli import with_appcontext
+
+import superset.utils.database as database_utils
+from superset import app, security_manager
+from superset.utils.celery import session_scope
+
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@with_appcontext
+def load_test_users() -> None:
+ """
+ Loads admin, alpha, and gamma user for testing purposes
+
+ Syncs permissions for those users/roles
+ """
+ print(Fore.GREEN + "Loading a set of users for unit tests")
+ load_test_users_run()
+
+
+def load_test_users_run() -> None:
+ """
+ Loads admin, alpha, and gamma user for testing purposes
+
+ Syncs permissions for those users/roles
+ """
+ if app.config["TESTING"]:
+
+ sm = security_manager
+
+ examples_db = database_utils.get_example_database()
+
+ examples_pv = sm.add_permission_view_menu("database_access", examples_db.perm)
+
+ sm.sync_role_definitions()
+ gamma_sqllab_role = sm.add_role("gamma_sqllab")
+ sm.add_permission_role(gamma_sqllab_role, examples_pv)
+
+ gamma_no_csv_role = sm.add_role("gamma_no_csv")
+ sm.add_permission_role(gamma_no_csv_role, examples_pv)
+
+ for role in ["Gamma", "sql_lab"]:
+ for perm in sm.find_role(role).permissions:
+ sm.add_permission_role(gamma_sqllab_role, perm)
+
+ if str(perm) != "can csv on Superset":
+ sm.add_permission_role(gamma_no_csv_role, perm)
+
+ users = (
+ ("admin", "Admin"),
+ ("gamma", "Gamma"),
+ ("gamma2", "Gamma"),
+ ("gamma_sqllab", "gamma_sqllab"),
+ ("alpha", "Alpha"),
+ ("gamma_no_csv", "gamma_no_csv"),
+ )
+ for username, role in users:
+ user = sm.find_user(username)
+ if not user:
+ sm.add_user(
+ username,
+ username,
+ "user",
+ username + "@fab.org",
+ sm.find_role(role),
+ password="general",
+ )
+ sm.get_session.commit()
+
+
+@click.command()
+@with_appcontext
+def alert() -> None:
+ """Run the alert scheduler loop"""
+ # this command is just for testing purposes
+ # pylint: disable=import-outside-toplevel
+ from superset.models.schedules import ScheduleType
+ from superset.tasks.schedules import schedule_window
+
+ click.secho("Processing one alert loop", fg="green")
+ with session_scope(nullpool=True) as session:
+ schedule_window(
+ report_type=ScheduleType.alert,
+ start_at=datetime.now() - timedelta(1000),
+ stop_at=datetime.now(),
+ resolution=6000,
+ session=session,
+ )
diff --git a/superset/cli/thumbnails.py b/superset/cli/thumbnails.py
new file mode 100755
index 0000000..5615d94
--- /dev/null
+++ b/superset/cli/thumbnails.py
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import logging
+from typing import Type, Union
+
+import click
+from celery.utils.abstract import CallableTask
+from flask.cli import with_appcontext
+
+from superset.extensions import db
+from superset.utils.urls import get_url_path
+
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@with_appcontext
+@click.option(
+ "--asynchronous",
+ "-a",
+ is_flag=True,
+ default=False,
+ help="Trigger commands to run remotely on a worker",
+)
+@click.option(
+ "--dashboards_only",
+ "-d",
+ is_flag=True,
+ default=False,
+ help="Only process dashboards",
+)
+@click.option(
+ "--charts_only", "-c", is_flag=True, default=False, help="Only process charts",
+)
+@click.option(
+ "--force",
+ "-f",
+ is_flag=True,
+ default=False,
+ help="Force refresh, even if previously cached",
+)
+@click.option("--model_id", "-i", multiple=True)
+def compute_thumbnails(
+ asynchronous: bool,
+ dashboards_only: bool,
+ charts_only: bool,
+ force: bool,
+ model_id: int,
+) -> None:
+ """Compute thumbnails"""
+ # pylint: disable=import-outside-toplevel
+ from superset.models.dashboard import Dashboard
+ from superset.models.slice import Slice
+ from superset.tasks.thumbnails import (
+ cache_chart_thumbnail,
+ cache_dashboard_thumbnail,
+ )
+
+ def compute_generic_thumbnail(
+ friendly_type: str,
+ model_cls: Union[Type[Dashboard], Type[Slice]],
+ model_id: int,
+ compute_func: CallableTask,
+ ) -> None:
+ query = db.session.query(model_cls)
+ if model_id:
+ query = query.filter(model_cls.id.in_(model_id))
+ dashboards = query.all()
+ count = len(dashboards)
+ for i, model in enumerate(dashboards):
+ if asynchronous:
+ func = compute_func.delay
+ action = "Triggering"
+ else:
+ func = compute_func
+ action = "Processing"
+ msg = f'{action} {friendly_type} "{model}" ({i+1}/{count})'
+ click.secho(msg, fg="green")
+ if friendly_type == "chart":
+ url = get_url_path(
+ "Superset.slice", slice_id=model.id, standalone="true"
+ )
+ else:
+ url = get_url_path("Superset.dashboard", dashboard_id_or_slug=model.id)
+ func(url, model.digest, force=force)
+
+ if not charts_only:
+ compute_generic_thumbnail(
+ "dashboard", Dashboard, model_id, cache_dashboard_thumbnail
+ )
+ if not dashboards_only:
+ compute_generic_thumbnail("chart", Slice, model_id, cache_chart_thumbnail)
diff --git a/superset/cli/update.py b/superset/cli/update.py
new file mode 100755
index 0000000..c3a7a2d
--- /dev/null
+++ b/superset/cli/update.py
@@ -0,0 +1,181 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+import logging
+import os
+import sys
+from datetime import datetime
+from typing import Optional
+
+import click
+from apispec import APISpec
+from apispec.ext.marshmallow import MarshmallowPlugin
+from flask import current_app
+from flask.cli import with_appcontext
+from flask_appbuilder import Model
+from flask_appbuilder.api import BaseApi
+from flask_appbuilder.api.manager import resolver
+
+import superset.utils.database as database_utils
+from superset.extensions import db
+from superset.utils.encrypt import SecretsMigrator
+
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@with_appcontext
+@click.option("--database_name", "-d", help="Database name to change")
+@click.option("--uri", "-u", help="Database URI to change")
+@click.option(
+ "--skip_create",
+ "-s",
+ is_flag=True,
+ default=False,
+ help="Create the DB if it doesn't exist",
+)
+def set_database_uri(database_name: str, uri: str, skip_create: bool) -> None:
+ """Updates a database connection URI"""
+ database_utils.get_or_create_db(database_name, uri, not skip_create)
+
+
+@click.command()
+@with_appcontext
+@click.option(
+ "--datasource",
+ "-d",
+ help="Specify which datasource name to load, if "
+ "omitted, all datasources will be refreshed",
+)
+@click.option(
+ "--merge",
+ "-m",
+ is_flag=True,
+ default=False,
+ help="Specify using 'merge' property during operation. " "Default value is False.",
+)
+def refresh_druid(datasource: str, merge: bool) -> None:
+ """Refresh druid datasources"""
+ # pylint: disable=import-outside-toplevel
+ from superset.connectors.druid.models import DruidCluster
+
+ session = db.session()
+
+ for cluster in session.query(DruidCluster).all():
+ try:
+ cluster.refresh_datasources(datasource_name=datasource, merge_flag=merge)
+ except Exception as ex: # pylint: disable=broad-except
+ print("Error while processing cluster '{}'\n{}".format(cluster, str(ex)))
+ logger.exception(ex)
+ cluster.metadata_last_refreshed = datetime.now()
+ print("Refreshed metadata from cluster " "[" + cluster.cluster_name + "]")
+ session.commit()
+
+
+@click.command()
+@with_appcontext
+def update_datasources_cache() -> None:
+ """Refresh sqllab datasources cache"""
+ # pylint: disable=import-outside-toplevel
+ from superset.models.core import Database
+
+ for database in db.session.query(Database).all():
+ if database.allow_multi_schema_metadata_fetch:
+ print("Fetching {} datasources ...".format(database.name))
+ try:
+ database.get_all_table_names_in_database(
+ force=True, cache=True, cache_timeout=24 * 60 * 60
+ )
+ database.get_all_view_names_in_database(
+ force=True, cache=True, cache_timeout=24 * 60 * 60
+ )
+ except Exception as ex: # pylint: disable=broad-except
+ print("{}".format(str(ex)))
+
+
+@click.command()
+@with_appcontext
+def sync_tags() -> None:
+ """Rebuilds special tags (owner, type, favorited by)."""
+ # pylint: disable=no-member
+ metadata = Model.metadata
+
+ # pylint: disable=import-outside-toplevel
+ from superset.common.tags import add_favorites, add_owners, add_types
+
+ add_types(db.engine, metadata)
+ add_owners(db.engine, metadata)
+ add_favorites(db.engine, metadata)
+
+
+@click.command()
+@with_appcontext
+def update_api_docs() -> None:
+ """Regenerate the openapi.json file in docs"""
+ superset_dir = os.path.abspath(os.path.dirname(__file__))
+ openapi_json = os.path.join(
+ superset_dir, "..", "docs", "src", "resources", "openapi.json"
+ )
+ api_version = "v1"
+
+ version_found = False
+ api_spec = APISpec(
+ title=current_app.appbuilder.app_name,
+ version=api_version,
+ openapi_version="3.0.2",
+ info=dict(description=current_app.appbuilder.app_name),
+ plugins=[MarshmallowPlugin(schema_name_resolver=resolver)],
+ servers=[{"url": "http://localhost:8088"}],
+ )
+ for base_api in current_app.appbuilder.baseviews:
+ if isinstance(base_api, BaseApi) and base_api.version == api_version:
+ base_api.add_api_spec(api_spec)
+ version_found = True
+ if version_found:
+ click.secho("Generating openapi.json", fg="green")
+ with open(openapi_json, "w") as outfile:
+ json.dump(api_spec.to_dict(), outfile, sort_keys=True, indent=2)
+ else:
+ click.secho("API version not found", err=True)
+
+
+@click.command()
+@with_appcontext
+@click.option(
+ "--previous_secret_key",
+ "-a",
+ required=False,
+ help="An optional previous secret key, if PREVIOUS_SECRET_KEY "
+ "is not set on the config",
+)
+def re_encrypt_secrets(previous_secret_key: Optional[str] = None) -> None:
+ previous_secret_key = previous_secret_key or current_app.config.get(
+ "PREVIOUS_SECRET_KEY"
+ )
+ if previous_secret_key is None:
+ click.secho("A previous secret key must be provided", err=True)
+ sys.exit(1)
+ secrets_migrator = SecretsMigrator(previous_secret_key=previous_secret_key)
+ try:
+ secrets_migrator.run()
+ except ValueError as exc:
+ click.secho(
+ f"An error occurred, "
+ f"probably an invalid previoud secret key was provided. Error:[{exc}]",
+ err=True,
+ )
+ sys.exit(1)
diff --git a/tests/integration_tests/cli_tests.py b/tests/integration_tests/cli_tests.py
index 5861627..baae90e 100644
--- a/tests/integration_tests/cli_tests.py
+++ b/tests/integration_tests/cli_tests.py
@@ -26,7 +26,7 @@ import pytest
import yaml
from freezegun import freeze_time
-import superset.cli
+import superset.cli.importexport
from superset import app
from tests.integration_tests.fixtures.birth_names_dashboard import (
load_birth_names_dashboard_with_slices,
@@ -53,14 +53,16 @@ def test_export_dashboards_original(app_context, fs):
Test that a JSON file is exported.
"""
# pylint: disable=reimported, redefined-outer-name
- import superset.cli # noqa: F811
+ import superset.cli.importexport # noqa: F811
# reload to define export_dashboards correctly based on the
# feature flags
- importlib.reload(superset.cli)
+ importlib.reload(superset.cli.importexport)
runner = app.test_cli_runner()
- response = runner.invoke(superset.cli.export_dashboards, ("-f", "dashboards.json"))
+ response = runner.invoke(
+ superset.cli.importexport.export_dashboards, ("-f", "dashboards.json")
+ )
assert response.exit_code == 0
assert Path("dashboards.json").exists()
@@ -77,15 +79,15 @@ def test_export_datasources_original(app_context, fs):
Test that a YAML file is exported.
"""
# pylint: disable=reimported, redefined-outer-name
- import superset.cli # noqa: F811
+ import superset.cli.importexport # noqa: F811
# reload to define export_dashboards correctly based on the
# feature flags
- importlib.reload(superset.cli)
+ importlib.reload(superset.cli.importexport)
runner = app.test_cli_runner()
response = runner.invoke(
- superset.cli.export_datasources, ("-f", "datasources.yaml")
+ superset.cli.importexport.export_datasources, ("-f", "datasources.yaml")
)
assert response.exit_code == 0
@@ -99,22 +101,22 @@ def test_export_datasources_original(app_context, fs):
@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
@mock.patch.dict(
- "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True
+ "superset.cli.lib.feature_flags", {"VERSIONED_EXPORT": True}, clear=True
)
def test_export_dashboards_versioned_export(app_context, fs):
"""
Test that a ZIP file is exported.
"""
# pylint: disable=reimported, redefined-outer-name
- import superset.cli # noqa: F811
+ import superset.cli.importexport # noqa: F811
# reload to define export_dashboards correctly based on the
# feature flags
- importlib.reload(superset.cli)
+ importlib.reload(superset.cli.importexport)
runner = app.test_cli_runner()
with freeze_time("2021-01-01T00:00:00Z"):
- response = runner.invoke(superset.cli.export_dashboards, ())
+ response = runner.invoke(superset.cli.importexport.export_dashboards, ())
assert response.exit_code == 0
assert Path("dashboard_export_20210101T000000.zip").exists()
@@ -123,7 +125,7 @@ def test_export_dashboards_versioned_export(app_context, fs):
@mock.patch.dict(
- "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True
+ "superset.cli.lib.feature_flags", {"VERSIONED_EXPORT": True}, clear=True
)
@mock.patch(
"superset.dashboards.commands.export.ExportDashboardsCommand.run",
@@ -138,37 +140,37 @@ def test_failing_export_dashboards_versioned_export(
caplog.set_level(logging.DEBUG)
# pylint: disable=reimported, redefined-outer-name
- import superset.cli # noqa: F811
+ import superset.cli.importexport # noqa: F811
# reload to define export_dashboards correctly based on the
# feature flags
- importlib.reload(superset.cli)
+ importlib.reload(superset.cli.importexport)
runner = app.test_cli_runner()
with freeze_time("2021-01-01T00:00:00Z"):
- response = runner.invoke(superset.cli.export_dashboards, ())
+ response = runner.invoke(superset.cli.importexport.export_dashboards, ())
assert_cli_fails_properly(response, caplog)
@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
@mock.patch.dict(
- "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True
+ "superset.cli.lib.feature_flags", {"VERSIONED_EXPORT": True}, clear=True
)
def test_export_datasources_versioned_export(app_context, fs):
"""
Test that a ZIP file is exported.
"""
# pylint: disable=reimported, redefined-outer-name
- import superset.cli # noqa: F811
+ import superset.cli.importexport # noqa: F811
# reload to define export_dashboards correctly based on the
# feature flags
- importlib.reload(superset.cli)
+ importlib.reload(superset.cli.importexport)
runner = app.test_cli_runner()
with freeze_time("2021-01-01T00:00:00Z"):
- response = runner.invoke(superset.cli.export_datasources, ())
+ response = runner.invoke(superset.cli.importexport.export_datasources, ())
assert response.exit_code == 0
assert Path("dataset_export_20210101T000000.zip").exists()
@@ -177,7 +179,7 @@ def test_export_datasources_versioned_export(app_context, fs):
@mock.patch.dict(
- "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True
+ "superset.cli.lib.feature_flags", {"VERSIONED_EXPORT": True}, clear=True
)
@mock.patch(
"superset.dashboards.commands.export.ExportDatasetsCommand.run",
@@ -190,21 +192,21 @@ def test_failing_export_datasources_versioned_export(
Test that failing to export ZIP file is done elegantly.
"""
# pylint: disable=reimported, redefined-outer-name
- import superset.cli # noqa: F811
+ import superset.cli.importexport # noqa: F811
# reload to define export_dashboards correctly based on the
# feature flags
- importlib.reload(superset.cli)
+ importlib.reload(superset.cli.importexport)
runner = app.test_cli_runner()
with freeze_time("2021-01-01T00:00:00Z"):
- response = runner.invoke(superset.cli.export_datasources, ())
+ response = runner.invoke(superset.cli.importexport.export_datasources, ())
assert_cli_fails_properly(response, caplog)
@mock.patch.dict(
- "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True
+ "superset.cli.lib.feature_flags", {"VERSIONED_EXPORT": True}, clear=True
)
@mock.patch("superset.dashboards.commands.importers.dispatcher.ImportDashboardsCommand")
def test_import_dashboards_versioned_export(import_dashboards_command, app_context, fs):
@@ -212,18 +214,20 @@ def test_import_dashboards_versioned_export(import_dashboards_command, app_conte
Test that both ZIP and JSON can be imported.
"""
# pylint: disable=reimported, redefined-outer-name
- import superset.cli # noqa: F811
+ import superset.cli.importexport # noqa: F811
# reload to define export_dashboards correctly based on the
# feature flags
- importlib.reload(superset.cli)
+ importlib.reload(superset.cli.importexport)
# write JSON file
with open("dashboards.json", "w") as fp:
fp.write('{"hello": "world"}')
runner = app.test_cli_runner()
- response = runner.invoke(superset.cli.import_dashboards, ("-p", "dashboards.json"))
+ response = runner.invoke(
+ superset.cli.importexport.import_dashboards, ("-p", "dashboards.json")
+ )
assert response.exit_code == 0
expected_contents = {"dashboards.json": '{"hello": "world"}'}
@@ -235,7 +239,9 @@ def test_import_dashboards_versioned_export(import_dashboards_command, app_conte
fp.write(b"hello: world")
runner = app.test_cli_runner()
- response = runner.invoke(superset.cli.import_dashboards, ("-p", "dashboards.zip"))
+ response = runner.invoke(
+ superset.cli.importexport.import_dashboards, ("-p", "dashboards.zip")
+ )
assert response.exit_code == 0
expected_contents = {"dashboard.yaml": "hello: world"}
@@ -243,7 +249,7 @@ def test_import_dashboards_versioned_export(import_dashboards_command, app_conte
@mock.patch.dict(
- "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True
+ "superset.cli.lib.feature_flags", {"VERSIONED_EXPORT": True}, clear=True
)
@mock.patch(
"superset.dashboards.commands.importers.dispatcher.ImportDashboardsCommand.run",
@@ -256,18 +262,20 @@ def test_failing_import_dashboards_versioned_export(
Test that failing to import either ZIP and JSON is done elegantly.
"""
# pylint: disable=reimported, redefined-outer-name
- import superset.cli # noqa: F811
+ import superset.cli.importexport # noqa: F811
# reload to define export_dashboards correctly based on the
# feature flags
- importlib.reload(superset.cli)
+ importlib.reload(superset.cli.importexport)
# write JSON file
with open("dashboards.json", "w") as fp:
fp.write('{"hello": "world"}')
runner = app.test_cli_runner()
- response = runner.invoke(superset.cli.import_dashboards, ("-p", "dashboards.json"))
+ response = runner.invoke(
+ superset.cli.importexport.import_dashboards, ("-p", "dashboards.json")
+ )
assert_cli_fails_properly(response, caplog)
@@ -277,13 +285,15 @@ def test_failing_import_dashboards_versioned_export(
fp.write(b"hello: world")
runner = app.test_cli_runner()
- response = runner.invoke(superset.cli.import_dashboards, ("-p", "dashboards.zip"))
+ response = runner.invoke(
+ superset.cli.importexport.import_dashboards, ("-p", "dashboards.zip")
+ )
assert_cli_fails_properly(response, caplog)
@mock.patch.dict(
- "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True
+ "superset.cli.lib.feature_flags", {"VERSIONED_EXPORT": True}, clear=True
)
@mock.patch("superset.datasets.commands.importers.dispatcher.ImportDatasetsCommand")
def test_import_datasets_versioned_export(import_datasets_command, app_context, fs):
@@ -291,18 +301,20 @@ def test_import_datasets_versioned_export(import_datasets_command, app_context,
Test that both ZIP and YAML can be imported.
"""
# pylint: disable=reimported, redefined-outer-name
- import superset.cli # noqa: F811
+ import superset.cli.importexport # noqa: F811
# reload to define export_datasets correctly based on the
# feature flags
- importlib.reload(superset.cli)
+ importlib.reload(superset.cli.importexport)
# write YAML file
with open("datasets.yaml", "w") as fp:
fp.write("hello: world")
runner = app.test_cli_runner()
- response = runner.invoke(superset.cli.import_datasources, ("-p", "datasets.yaml"))
+ response = runner.invoke(
+ superset.cli.importexport.import_datasources, ("-p", "datasets.yaml")
+ )
assert response.exit_code == 0
expected_contents = {"datasets.yaml": "hello: world"}
@@ -314,7 +326,9 @@ def test_import_datasets_versioned_export(import_datasets_command, app_context,
fp.write(b"hello: world")
runner = app.test_cli_runner()
- response = runner.invoke(superset.cli.import_datasources, ("-p", "datasets.zip"))
+ response = runner.invoke(
+ superset.cli.importexport.import_datasources, ("-p", "datasets.zip")
+ )
assert response.exit_code == 0
expected_contents = {"dataset.yaml": "hello: world"}
@@ -322,7 +336,7 @@ def test_import_datasets_versioned_export(import_datasets_command, app_context,
@mock.patch.dict(
- "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True
+ "superset.cli.lib.feature_flags", {"VERSIONED_EXPORT": True}, clear=True
)
@mock.patch(
"superset.datasets.commands.importers.dispatcher.ImportDatasetsCommand.run",
@@ -335,18 +349,20 @@ def test_failing_import_datasets_versioned_export(
Test that failing to import either ZIP or YAML is done elegantly.
"""
# pylint: disable=reimported, redefined-outer-name
- import superset.cli # noqa: F811
+ import superset.cli.importexport # noqa: F811
# reload to define export_datasets correctly based on the
# feature flags
- importlib.reload(superset.cli)
+ importlib.reload(superset.cli.importexport)
# write YAML file
with open("datasets.yaml", "w") as fp:
fp.write("hello: world")
runner = app.test_cli_runner()
- response = runner.invoke(superset.cli.import_datasources, ("-p", "datasets.yaml"))
+ response = runner.invoke(
+ superset.cli.importexport.import_datasources, ("-p", "datasets.yaml")
+ )
assert_cli_fails_properly(response, caplog)
@@ -356,6 +372,8 @@ def test_failing_import_datasets_versioned_export(
fp.write(b"hello: world")
runner = app.test_cli_runner()
- response = runner.invoke(superset.cli.import_datasources, ("-p", "datasets.zip"))
+ response = runner.invoke(
+ superset.cli.importexport.import_datasources, ("-p", "datasets.zip")
+ )
assert_cli_fails_properly(response, caplog)
diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py
index d143f94..fee13c8 100644
--- a/tests/integration_tests/conftest.py
+++ b/tests/integration_tests/conftest.py
@@ -50,7 +50,7 @@ def setup_sample_data() -> Any:
with app.app_context():
setup_presto_if_needed()
- from superset.cli import load_test_users_run
+ from superset.cli.test import load_test_users_run
load_test_users_run()