You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by hu...@apache.org on 2021/05/24 18:52:15 UTC
[superset] branch hugh/bg-validation-db-modal created (now b5f3855)
This is an automated email from the ASF dual-hosted git repository.
hugh pushed a change to branch hugh/bg-validation-db-modal
in repository https://gitbox.apache.org/repos/asf/superset.git.
at b5f3855 add big query to database connection form
This branch includes the following new commits:
new 29ab6c8 fix merge conflicts
new b5f3855 add big query to database connection form
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[superset] 02/02: add big query to database connection form
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hugh pushed a commit to branch hugh/bg-validation-db-modal
in repository https://gitbox.apache.org/repos/asf/superset.git
commit b5f385517c9a43cdecfbe57062a511ec6e4a9b66
Author: hughhhh <hu...@gmail.com>
AuthorDate: Mon May 24 14:50:56 2021 -0400
add big query to database connection form
---
superset-frontend/src/common/components/index.tsx | 1 +
.../DatabaseModal/DatabaseConnectionForm.tsx | 32 +++++++++++++++++++++-
2 files changed, 32 insertions(+), 1 deletion(-)
diff --git a/superset-frontend/src/common/components/index.tsx b/superset-frontend/src/common/components/index.tsx
index bf17aac..c0cf44a 100644
--- a/superset-frontend/src/common/components/index.tsx
+++ b/superset-frontend/src/common/components/index.tsx
@@ -54,6 +54,7 @@ export {
Tag,
Tabs,
Tooltip,
+ Upload,
Input as AntdInput,
} from 'antd';
export { Card as AntdCard } from 'antd';
diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx
index 1eeab48..bde8009 100644
--- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx
+++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx
@@ -16,10 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-import React, { FormEvent } from 'react';
+import React, { FormEvent, useState} from 'react';
import { SupersetTheme } from '@superset-ui/core';
import { InputProps } from 'antd/lib/input';
import ValidatedInput from 'src/components/Form/LabeledErrorBoundInput';
+import { Button, Select, Input, Upload } from 'src/common/components';
import {
StyledFormHeader,
formScrollableStyles,
@@ -34,6 +35,7 @@ export const FormFieldOrder = [
'username',
'password',
'database_name',
+ 'credentials_info',
];
interface FieldPropTypes {
@@ -43,6 +45,33 @@ interface FieldPropTypes {
};
}
+const CredentialInfo = ({ required, changeMethods }: FieldPropTypes) => {
+ const [uploadOption, setUploadOption] = useState<string>('upload');
+ return (
+ <>
+ <Select
+ defaultValue={'file'}
+ style={{ width: '100%' }}
+ onChange={value => {
+ setUploadOption(value);
+ }}
+ >
+ <Select value="file">Upload JSON file</Select>
+ <Select value="paste">Copy and Paste JSON credentials</Select>
+ </Select>
+ {uploadOption === 'paste' ? (
+ <div className="input-container">
+ <Input rows={4}/>
+ </div>
+ ) : (
+ <Upload>
+ <Button>Click to Upload</Button>
+ </Upload>
+ )}
+ </>
+ );
+};
+
const hostField = ({ required, changeMethods }: FieldPropTypes) => (
<ValidatedInput
id="host"
@@ -121,6 +150,7 @@ const FORM_FIELD_MAP = {
username: usernameField,
password: passwordField,
database_name: displayField,
+ credentials_info: CredentialInfo,
};
const DatabaseConnectionForm = ({
[superset] 01/02: fix merge conflicts
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hugh pushed a commit to branch hugh/bg-validation-db-modal
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 29ab6c866b9d7a69fb5154a35d9d7720bd19c2e2
Author: Hugh A. Miles II <hu...@gmail.com>
AuthorDate: Sun May 23 12:45:48 2021 -0400
fix merge conflicts
---
superset/databases/api.py | 11 ++++--
superset/databases/commands/validate.py | 12 +++++-
superset/databases/schemas.py | 34 ++++++++++++-----
superset/db_engine_specs/base.py | 6 ++-
superset/db_engine_specs/bigquery.py | 66 +++++++++++++++++++++++++++++++++
superset/models/core.py | 1 +
tests/databases/api_tests.py | 18 +++++++++
tests/db_engine_specs/postgres_tests.py | 5 ++-
8 files changed, 135 insertions(+), 18 deletions(-)
diff --git a/superset/databases/api.py b/superset/databases/api.py
index 3f506e0..0b0d297 100644
--- a/superset/databases/api.py
+++ b/superset/databases/api.py
@@ -66,7 +66,6 @@ from superset.databases.schemas import (
)
from superset.databases.utils import get_table_metadata
from superset.db_engine_specs import get_available_engine_specs
-from superset.db_engine_specs.base import BasicParametersMixin
from superset.exceptions import InvalidPayloadFormatError, InvalidPayloadSchemaError
from superset.extensions import security_manager
from superset.models.core import Database
@@ -909,11 +908,15 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
"preferred": engine_spec.engine in preferred_databases,
}
- if issubclass(engine_spec, BasicParametersMixin):
- payload["parameters"] = engine_spec.parameters_json_schema()
+ if hasattr(engine_spec, "parameters_json_schema") and hasattr(
+ engine_spec, "sqlalchemy_uri_placeholder"
+ ):
+ payload[
+ "parameters"
+ ] = engine_spec.parameters_json_schema() # type: ignore
payload[
"sqlalchemy_uri_placeholder"
- ] = engine_spec.sqlalchemy_uri_placeholder
+ ] = engine_spec.sqlalchemy_uri_placeholder # type: ignore
available_databases.append(payload)
diff --git a/superset/databases/commands/validate.py b/superset/databases/commands/validate.py
index e062102..93e32ef 100644
--- a/superset/databases/commands/validate.py
+++ b/superset/databases/commands/validate.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import json
from contextlib import closing
from typing import Any, Dict, Optional
@@ -81,9 +82,16 @@ class ValidateDatabaseParametersCommand(BaseCommand):
if errors:
raise InvalidParametersError(errors)
+ serialized_encrypted_extra = self._properties.get("encrypted_extra", "{}")
+ try:
+ encrypted_extra = json.loads(serialized_encrypted_extra)
+ except json.decoder.JSONDecodeError:
+ encrypted_extra = {}
+
# try to connect
sqlalchemy_uri = engine_spec.build_sqlalchemy_uri(
- self._properties["parameters"] # type: ignore
+ self._properties["parameters"], # type: ignore
+ encrypted_extra,
)
if self._model and sqlalchemy_uri == self._model.safe_sqlalchemy_uri():
sqlalchemy_uri = self._model.sqlalchemy_uri_decrypted
@@ -91,7 +99,7 @@ class ValidateDatabaseParametersCommand(BaseCommand):
server_cert=self._properties.get("server_cert", ""),
extra=self._properties.get("extra", "{}"),
impersonate_user=self._properties.get("impersonate_user", False),
- encrypted_extra=self._properties.get("encrypted_extra", "{}"),
+ encrypted_extra=serialized_encrypted_extra,
)
database.set_sqlalchemy_uri(sqlalchemy_uri)
database.db_engine_spec.mutate_db_for_connection_test(database)
diff --git a/superset/databases/schemas.py b/superset/databases/schemas.py
index 924ac94..2ebddd7 100644
--- a/superset/databases/schemas.py
+++ b/superset/databases/schemas.py
@@ -247,6 +247,12 @@ class DatabaseParametersSchemaMixin:
the constructed SQLAlchemy URI to be passed.
"""
parameters = data.pop("parameters", None)
+ serialized_encrypted_extra = data.get("encrypted_extra", "{}")
+ try:
+ encrypted_extra = json.loads(serialized_encrypted_extra)
+ except json.decoder.JSONDecodeError:
+ encrypted_extra = {}
+
if parameters:
if "engine" not in parameters:
raise ValidationError(
@@ -265,18 +271,14 @@ class DatabaseParametersSchemaMixin:
[_('Engine "%(engine)s" is not a valid engine.', engine=engine,)]
)
engine_spec = engine_specs[engine]
- if not issubclass(engine_spec, BasicParametersMixin):
- raise ValidationError(
- [
- _(
- 'Engine spec "%(engine_spec)s" does not support '
- "being configured via individual parameters.",
- engine_spec=engine_spec.__name__,
- )
- ]
+
+ if hasattr(engine_spec, "build_sqlalchemy_uri"):
+ data[
+ "sqlalchemy_uri"
+ ] = engine_spec.build_sqlalchemy_uri( # type: ignore
+ parameters, encrypted_extra
)
- data["sqlalchemy_uri"] = engine_spec.build_sqlalchemy_uri(parameters)
return data
@@ -553,3 +555,15 @@ class ImportV1DatabaseSchema(Schema):
password = make_url(uri).password
if password == PASSWORD_MASK and data.get("password") is None:
raise ValidationError("Must provide a password for the database")
+
+
+class EncryptedField(fields.String):
+ pass
+
+
+def encrypted_field_properties(self, field: Any, **_) -> Dict[str, Any]: # type: ignore
+ ret = {}
+ if isinstance(field, EncryptedField):
+ if self.openapi_version.major > 2:
+ ret["x-encrypted-extra"] = True
+ return ret
diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py
index fe0e434..97321b8 100644
--- a/superset/db_engine_specs/base.py
+++ b/superset/db_engine_specs/base.py
@@ -1348,7 +1348,11 @@ class BasicParametersMixin:
encryption_parameters: Dict[str, str] = {}
@classmethod
- def build_sqlalchemy_uri(cls, parameters: BasicParametersType) -> str:
+ def build_sqlalchemy_uri(
+ cls,
+ parameters: BasicParametersType,
+ encryted_extra: Optional[Dict[str, str]] = None,
+ ) -> str:
query = parameters.get("query", {})
if parameters.get("encryption"):
if not cls.encryption_parameters:
diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py
index 9c52b44..a7ce77c 100644
--- a/superset/db_engine_specs/bigquery.py
+++ b/superset/db_engine_specs/bigquery.py
@@ -19,12 +19,18 @@ from datetime import datetime
from typing import Any, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
import pandas as pd
+from apispec import APISpec
+from apispec.ext.marshmallow import MarshmallowPlugin
from flask_babel import gettext as __
+from marshmallow import Schema
from sqlalchemy import literal_column
from sqlalchemy.sql.expression import ColumnClause
+from typing_extensions import TypedDict
+from superset.databases.schemas import encrypted_field_properties, EncryptedField
from superset.db_engine_specs.base import BaseEngineSpec
from superset.errors import SupersetErrorType
+from superset.exceptions import SupersetGenericDBErrorException
from superset.sql_parse import Table
from superset.utils import core as utils
from superset.utils.hashing import md5_sha_from_str
@@ -38,6 +44,18 @@ CONNECTION_DATABASE_PERMISSIONS_REGEX = re.compile(
+ "permission in project (?P<project>.+?)"
)
+ma_plugin = MarshmallowPlugin()
+
+
+class BigQueryParametersSchema(Schema):
+ credentials_info = EncryptedField(
+ description="Contents of BigQuery JSON credentials.",
+ )
+
+
+class BigQueryParametersType(TypedDict):
+ credentials_info: Dict[str, Any]
+
class BigQueryEngineSpec(BaseEngineSpec):
"""Engine spec for Google's BigQuery
@@ -48,6 +66,10 @@ class BigQueryEngineSpec(BaseEngineSpec):
engine_name = "Google BigQuery"
max_column_name_length = 128
+ parameters_schema = BigQueryParametersSchema()
+ drivername = engine
+ sqlalchemy_uri_placeholder = "bigquery://{project_id}"
+
# BigQuery doesn't maintain context when running multiple statements in the
# same cursor, so we need to run all statements at once
run_multiple_statements_as_one = True
@@ -282,3 +304,47 @@ class BigQueryEngineSpec(BaseEngineSpec):
to_gbq_kwargs[key] = to_sql_kwargs[key]
pandas_gbq.to_gbq(df, **to_gbq_kwargs)
+
+ @classmethod
+ def build_sqlalchemy_uri(
+ cls, _: BigQueryParametersType, encrypted_extra: Optional[Dict[str, str]] = None
+ ) -> str:
+ if encrypted_extra:
+ project_id = encrypted_extra.get("project_id")
+ return f"{cls.drivername}://{project_id}"
+
+ raise SupersetGenericDBErrorException(
+ message="Big Query encrypted_extra is not available.",
+ )
+
+ @classmethod
+ def get_parameters_from_uri(
+ cls, _: str, encrypted_extra: Optional[Dict[str, str]] = None
+ ) -> Any:
+ # BigQuery doesn't have parameters
+ if encrypted_extra:
+ return encrypted_extra
+
+ raise SupersetGenericDBErrorException(
+ message="Big Query encrypted_extra is not available.",
+ )
+
+ @classmethod
+ def parameters_json_schema(cls) -> Any:
+ """
+ Return configuration parameters as OpenAPI.
+ """
+ if not cls.parameters_schema:
+ return None
+
+ spec = APISpec(
+ title="Database Parameters",
+ version="1.0.0",
+ openapi_version="3.0.0",
+ plugins=[ma_plugin],
+ )
+
+ ma_plugin.init_spec(spec)
+ ma_plugin.converter.add_attribute_function(encrypted_field_properties)
+ spec.components.schema(cls.__name__, schema=cls.parameters_schema)
+ return spec.to_dict()["components"]["schemas"][cls.__name__]
diff --git a/superset/models/core.py b/superset/models/core.py
index 1868af6..540974f 100755
--- a/superset/models/core.py
+++ b/superset/models/core.py
@@ -241,6 +241,7 @@ class Database(
def parameters(self) -> Dict[str, Any]:
# Build parameters if db_engine_spec is a subclass of BasicParametersMixin
parameters = {"engine": self.backend}
+
if hasattr(self.db_engine_spec, "parameters_schema") and hasattr(
self.db_engine_spec, "get_parameters_from_uri"
):
diff --git a/tests/databases/api_tests.py b/tests/databases/api_tests.py
index 7f53c57..1868f95 100644
--- a/tests/databases/api_tests.py
+++ b/tests/databases/api_tests.py
@@ -36,6 +36,7 @@ from superset import db, security_manager
from superset.connectors.sqla.models import SqlaTable
from superset.db_engine_specs.mysql import MySQLEngineSpec
from superset.db_engine_specs.postgres import PostgresEngineSpec
+from superset.db_engine_specs.bigquery import BigQueryEngineSpec
from superset.db_engine_specs.hana import HanaEngineSpec
from superset.errors import SupersetError
from superset.models.core import Database, ConfigurationMethod
@@ -1373,6 +1374,7 @@ class TestDatabaseApi(SupersetTestCase):
app.config = {"PREFERRED_DATABASES": ["postgresql"]}
get_available_engine_specs.return_value = [
PostgresEngineSpec,
+ BigQueryEngineSpec,
HanaEngineSpec,
]
@@ -1428,6 +1430,22 @@ class TestDatabaseApi(SupersetTestCase):
"preferred": True,
"sqlalchemy_uri_placeholder": "postgresql+psycopg2://user:password@host:port/dbname[?key=value&key=value...]",
},
+ {
+ "engine": "bigquery",
+ "name": "Google BigQuery",
+ "parameters": {
+ "properties": {
+ "credentials_info": {
+ "description": "Contents of BigQuery JSON credentials.",
+ "type": "string",
+ "x-encrypted-extra": True,
+ }
+ },
+ "type": "object",
+ },
+ "preferred": False,
+ "sqlalchemy_uri_placeholder": "bigquery://{project_id}",
+ },
{"engine": "hana", "name": "SAP HANA", "preferred": False},
]
}
diff --git a/tests/db_engine_specs/postgres_tests.py b/tests/db_engine_specs/postgres_tests.py
index 6f6fa54..135621b 100644
--- a/tests/db_engine_specs/postgres_tests.py
+++ b/tests/db_engine_specs/postgres_tests.py
@@ -432,7 +432,10 @@ def test_base_parameters_mixin():
"query": {"foo": "bar"},
"encryption": True,
}
- sqlalchemy_uri = PostgresEngineSpec.build_sqlalchemy_uri(parameters)
+ encrypted_extra = None
+ sqlalchemy_uri = PostgresEngineSpec.build_sqlalchemy_uri(
+ parameters, encrypted_extra
+ )
assert sqlalchemy_uri == (
"postgresql+psycopg2://username:password@localhost:5432/dbname?"
"foo=bar&sslmode=verify-ca"