You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by hu...@apache.org on 2021/05/18 21:21:08 UTC
[superset] branch hugh/bg-form-2 created (now 6bdf47c)
This is an automated email from the ASF dual-hosted git repository.
hugh pushed a change to branch hugh/bg-form-2
in repository https://gitbox.apache.org/repos/asf/superset.git.
at 6bdf47c pull bg form changes
This branch includes the following new commits:
new 6bdf47c pull bg form changes
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[superset] 01/01: pull bg form changes
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hugh pushed a commit to branch hugh/bg-form-2
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 6bdf47cf0d2da6a9198088ea1dd190eda16c451f
Author: hughhhh <hu...@gmail.com>
AuthorDate: Tue May 18 17:19:26 2021 -0400
pull bg form changes
---
superset/db_engine_specs/bigquery.py | 48 ++++++++++++++++++++-
tests/databases/api_tests.py | 83 +++++++++++++++++++++---------------
2 files changed, 95 insertions(+), 36 deletions(-)
diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py
index 9c52b44..edf0410 100644
--- a/superset/db_engine_specs/bigquery.py
+++ b/superset/db_engine_specs/bigquery.py
@@ -20,8 +20,10 @@ from typing import Any, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
import pandas as pd
from flask_babel import gettext as __
+from marshmallow import fields, Schema
from sqlalchemy import literal_column
from sqlalchemy.sql.expression import ColumnClause
+from typing_extensions import TypedDict
from superset.db_engine_specs.base import BaseEngineSpec
from superset.errors import SupersetErrorType
@@ -39,7 +41,30 @@ CONNECTION_DATABASE_PERMISSIONS_REGEX = re.compile(
)
-class BigQueryEngineSpec(BaseEngineSpec):
+class BigQueryCredsJsonSchema(Schema):
+ type = fields.String()
+ project_id = fields.String()
+ private_key_id = fields.String()
+ private_key = fields.String()
+ client_email = fields.String()
+ client_id = fields.String()
+ auth_uri = fields.String()
+ token_uri = fields.String()
+ auth_provider_x509_cert_url = fields.String()
+ client_x509_cert_url = fields.String()
+
+
+class BigQueryParametersSchema(Schema):
+ credentials_json = fields.Nested(
+ BigQueryCredsJsonSchema, description=__("Credentials for BigQuery"),
+ )
+
+
+class BigQueryParametersType(TypedDict):
+ credentials_json: Dict[str, Any]
+
+
+class BigQueryEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
"""Engine spec for Google's BigQuery
As contributed by @mxmzdlv on issue #945"""
@@ -48,6 +73,11 @@ class BigQueryEngineSpec(BaseEngineSpec):
engine_name = "Google BigQuery"
max_column_name_length = 128
+ # Mixin overrides
+ parameters_schema = BigQueryParametersSchema()
+ drivername = engine
+ sqlalchemy_uri_placeholder = "bigquery://{project_id}"
+
# BigQuery doesn't maintain context when running multiple statements in the
# same cursor, so we need to run all statements at once
run_multiple_statements_as_one = True
@@ -282,3 +312,19 @@ class BigQueryEngineSpec(BaseEngineSpec):
to_gbq_kwargs[key] = to_sql_kwargs[key]
pandas_gbq.to_gbq(df, **to_gbq_kwargs)
+
+ @classmethod
+ def build_sqlalchemy_url(cls, parameters: BigQueryParametersType) -> str:
+ project_id = (
+ parameters.get("credentials_json", {})
+ .get("credentials_info", {})
+ .get("project_id")
+ )
+
+ return f"{cls.drivername}://{project_id}"
+
+ @classmethod
+ def get_parameters_from_uri(cls, uri: str) -> Optional[BigQueryParametersType]:
+ # We might need to add a special case for bigquery since
+ # we are relying on the json credentials
+ return None
diff --git a/tests/databases/api_tests.py b/tests/databases/api_tests.py
index d7404f5..2e1d9f7 100644
--- a/tests/databases/api_tests.py
+++ b/tests/databases/api_tests.py
@@ -36,6 +36,7 @@ from superset import db, security_manager
from superset.connectors.sqla.models import SqlaTable
from superset.db_engine_specs.mysql import MySQLEngineSpec
from superset.db_engine_specs.postgres import PostgresEngineSpec
+from superset.db_engine_specs.bigquery import BigQueryEngineSpec
from superset.errors import SupersetError
from superset.models.core import Database, ConfigurationMethod
from superset.models.reports import ReportSchedule, ReportScheduleType
@@ -275,36 +276,35 @@ class TestDatabaseApi(SupersetTestCase):
}
assert rv.status_code == 400
- # add this test back in when config method becomes required for creation.
- # def test_create_database_no_configuration_method(self):
- # """
- # Database API: Test create with no config method.
- # """
- # extra = {
- # "metadata_params": {},
- # "engine_params": {},
- # "metadata_cache_timeout": {},
- # "schemas_allowed_for_csv_upload": [],
- # }
-
- # self.login(username="admin")
- # example_db = get_example_database()
- # if example_db.backend == "sqlite":
- # return
- # database_data = {
- # "database_name": "test-create-database",
- # "sqlalchemy_uri": example_db.sqlalchemy_uri_decrypted,
- # "server_cert": None,
- # "extra": json.dumps(extra),
- # }
-
- # uri = "api/v1/database/"
- # rv = self.client.post(uri, json=database_data)
- # response = json.loads(rv.data.decode("utf-8"))
- # assert response == {
- # "message": {"configuration_method": ["Missing data for required field."]}
- # }
- # assert rv.status_code == 400
+ def test_create_database_no_configuration_method(self):
+ """
+ Database API: Test create with no config method.
+ """
+ extra = {
+ "metadata_params": {},
+ "engine_params": {},
+ "metadata_cache_timeout": {},
+ "schemas_allowed_for_csv_upload": [],
+ }
+
+ self.login(username="admin")
+ example_db = get_example_database()
+ if example_db.backend == "sqlite":
+ return
+ database_data = {
+ "database_name": "test-create-database",
+ "sqlalchemy_uri": example_db.sqlalchemy_uri_decrypted,
+ "server_cert": None,
+ "extra": json.dumps(extra),
+ }
+
+ uri = "api/v1/database/"
+ rv = self.client.post(uri, json=database_data)
+ response = json.loads(rv.data.decode("utf-8"))
+ assert response == {
+ "message": {"configuration_method": ["Missing data for required field."]}
+ }
+ assert rv.status_code == 400
def test_create_database_server_cert_validate(self):
"""
@@ -1368,10 +1368,11 @@ class TestDatabaseApi(SupersetTestCase):
@mock.patch("superset.databases.api.get_available_engine_specs")
@mock.patch("superset.databases.api.app")
def test_available(self, app, get_available_engine_specs):
- app.config = {"PREFERRED_DATABASES": ["postgresql"]}
+ app.config = {"PREFERRED_DATABASES": ["postgresql", "bigquery"]}
get_available_engine_specs.return_value = [
MySQLEngineSpec,
PostgresEngineSpec,
+ BigQueryEngineSpec,
]
self.login(username="admin")
@@ -1392,10 +1393,6 @@ class TestDatabaseApi(SupersetTestCase):
"description": "Database name",
"type": "string",
},
- "encryption": {
- "description": "Use an encrypted connection to the database",
- "type": "boolean",
- },
"host": {
"description": "Hostname or IP address",
"type": "string",
@@ -1427,6 +1424,22 @@ class TestDatabaseApi(SupersetTestCase):
"preferred": True,
"sqlalchemy_uri_placeholder": "postgresql+psycopg2://user:password@host:port/dbname[?key=value&key=value...]",
},
+ {
+ "engine": "bigquery",
+ "name": "Google BigQuery",
+ "parameters": {
+ "properties": {
+ "credentials_json": {
+ "additionalProperties": {},
+ "description": "credentials for bigquery",
+ "type": "object",
+ }
+ },
+ "type": "object",
+ },
+ "preferred": true,
+ "sqlalchemy_uri_placeholder": "bigquery://{project_id}",
+ },
{"engine": "mysql", "name": "MySQL", "preferred": False},
]
}