You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by be...@apache.org on 2023/08/09 02:39:57 UTC
[superset] branch upload_gsheets updated (1d09b36835 -> 03a01023cf)
This is an automated email from the ASF dual-hosted git repository.
beto pushed a change to branch upload_gsheets
in repository https://gitbox.apache.org/repos/asf/superset.git
omit 1d09b36835 Fix tests
new 622aff25b1 Fix tests
new 03a01023cf Add dependency
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1d09b36835)
\
N -- N -- N refs/heads/upload_gsheets (03a01023cf)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
requirements/development.txt | 2 +-
requirements/testing.in | 2 +-
requirements/testing.txt | 7 ++++++-
tests/unit_tests/db_engine_specs/test_gsheets.py | 4 ++--
4 files changed, 10 insertions(+), 5 deletions(-)
[superset] 02/02: Add dependency
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
beto pushed a commit to branch upload_gsheets
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 03a01023cf3e9dbea4fc97a575d7912300de2fee
Author: Beto Dealmeida <ro...@dealmeida.net>
AuthorDate: Tue Aug 8 19:38:22 2023 -0700
Add dependency
---
requirements/development.txt | 2 +-
requirements/testing.in | 2 +-
requirements/testing.txt | 7 ++++++-
3 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/requirements/development.txt b/requirements/development.txt
index bba99045cc..7155b2e1a5 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -106,7 +106,7 @@ pylint==2.17.4
# via -r requirements/development.in
python-ldap==3.4.3
# via -r requirements/development.in
-requests==2.30.0
+requests==2.31.0
# via
# pydruid
# tableschema
diff --git a/requirements/testing.in b/requirements/testing.in
index 856c5272dc..b991be1040 100644
--- a/requirements/testing.in
+++ b/requirements/testing.in
@@ -16,7 +16,7 @@
#
-r development.in
-r integration.in
--e file:.[bigquery,hive,presto,prophet,trino]
+-e file:.[bigquery,hive,presto,prophet,trino,gsheets]
docker
flask-testing
freezegun
diff --git a/requirements/testing.txt b/requirements/testing.txt
index 283b4c9fcd..c8a3221b45 100644
--- a/requirements/testing.txt
+++ b/requirements/testing.txt
@@ -1,4 +1,4 @@
-# SHA1:78fe89f88adf34ac75513d363d7d9d0b5cc8cd1c
+# SHA1:78d0270a4f583095e0587aa21f57fc2ff7fe8b84
#
# This file is autogenerated by pip-compile-multi
# To update, run:
@@ -12,6 +12,8 @@
# -r requirements/base.in
# -r requirements/development.in
# -r requirements/testing.in
+apsw==3.42.0.1
+ # via shillelagh
cmdstanpy==1.1.0
# via prophet
contourpy==1.0.7
@@ -50,6 +52,7 @@ google-auth==2.17.3
# google-cloud-core
# pandas-gbq
# pydata-google-auth
+ # shillelagh
# sqlalchemy-bigquery
google-auth-oauthlib==1.0.0
# via
@@ -142,6 +145,8 @@ rfc3339-validator==0.1.4
# via openapi-schema-validator
rsa==4.9
# via google-auth
+shillelagh[gsheetsapi]==1.2.6
+ # via apache-superset
sqlalchemy-bigquery==1.6.1
# via apache-superset
statsd==4.0.1
[superset] 01/02: Fix tests
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
beto pushed a commit to branch upload_gsheets
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 622aff25b115338e838e6e7ba257695c4b9d0d0b
Author: Beto Dealmeida <ro...@dealmeida.net>
AuthorDate: Tue Aug 8 19:06:19 2023 -0700
Fix tests
---
superset/db_engine_specs/gsheets.py | 18 +++++++++++-------
tests/unit_tests/db_engine_specs/test_gsheets.py | 4 ++--
2 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/superset/db_engine_specs/gsheets.py b/superset/db_engine_specs/gsheets.py
index b75742254a..59d9a6bcf8 100644
--- a/superset/db_engine_specs/gsheets.py
+++ b/superset/db_engine_specs/gsheets.py
@@ -31,7 +31,6 @@ from flask_babel import gettext as __
from marshmallow import fields, Schema
from marshmallow.exceptions import ValidationError
from requests import Session
-from shillelagh.backends.apsw.dialects.base import get_adapter_for_table_name
from sqlalchemy.engine import create_engine
from sqlalchemy.engine.url import URL
from typing_extensions import TypedDict
@@ -49,7 +48,10 @@ if TYPE_CHECKING:
_logger = logging.getLogger()
-EXAMPLE_GSHEETS_URL = "https://docs.google.com/spreadsheets/d/1LcWZMsdCl92g7nA-D6qGRqg1T5TiHyuKJUY1u9XAnsk/edit#gid=0"
+EXAMPLE_GSHEETS_URL = (
+ "https://docs.google.com/spreadsheets/d/"
+ "1LcWZMsdCl92g7nA-D6qGRqg1T5TiHyuKJUY1u9XAnsk/edit#gid=0"
+)
SYNTAX_ERROR_REGEX = re.compile('SQLError: near "(?P<server_error>.*?)": syntax error')
@@ -138,8 +140,8 @@ class GSheetsEngineSpec(ShillelaghEngineSpec):
def build_sqlalchemy_uri(
cls,
_: GSheetsParametersType,
- encrypted_extra: None
- | (dict[str, Any]) = None, # pylint: disable=unused-argument
+ encrypted_extra: None # pylint: disable=unused-argument
+ | (dict[str, Any]) = None,
) -> str:
return "gsheets://"
@@ -338,7 +340,7 @@ class GSheetsEngineSpec(ShillelaghEngineSpec):
return payload
@classmethod
- def df_to_sql(
+ def df_to_sql( # pylint: disable=too-many-locals
cls,
database: Database,
table: Table,
@@ -358,6 +360,8 @@ class GSheetsEngineSpec(ShillelaghEngineSpec):
cells in the existing sheet before uploading the new data. Appending to an
existing table is not supported because we can't ensure that the schemas match.
"""
+ from shillelagh.backends.apsw.dialects.base import get_adapter_for_table_name
+
# grab the existing catalog, if any
extra = database.get_extra()
engine_params = extra.setdefault("engine_params", {})
@@ -387,8 +391,8 @@ class GSheetsEngineSpec(ShillelaghEngineSpec):
# clear existing sheet, or create a new one
if spreadsheet_url:
- spreadsheet_id = adapter._spreadsheet_id
- range_ = adapter._sheet_name
+ spreadsheet_id = adapter._spreadsheet_id # pylint: disable=protected-access
+ range_ = adapter._sheet_name # pylint: disable=protected-access
url = (
"https://sheets.googleapis.com/v4/spreadsheets/"
f"{spreadsheet_id}/values/{range_}:clear"
diff --git a/tests/unit_tests/db_engine_specs/test_gsheets.py b/tests/unit_tests/db_engine_specs/test_gsheets.py
index 6f580f2674..7d7348c1a3 100644
--- a/tests/unit_tests/db_engine_specs/test_gsheets.py
+++ b/tests/unit_tests/db_engine_specs/test_gsheets.py
@@ -321,7 +321,7 @@ def test_upload_new(mocker: MockFixture) -> None:
mocker.patch("superset.db_engine_specs.gsheets.db")
get_adapter_for_table_name = mocker.patch(
- "superset.db_engine_specs.gsheets.get_adapter_for_table_name"
+ "shillelagh.backends.apsw.dialects.base.get_adapter_for_table_name"
)
session = get_adapter_for_table_name()._get_session()
session.post().json.return_value = {
@@ -350,7 +350,7 @@ def test_upload_existing(mocker: MockFixture) -> None:
mocker.patch("superset.db_engine_specs.gsheets.db")
get_adapter_for_table_name = mocker.patch(
- "superset.db_engine_specs.gsheets.get_adapter_for_table_name"
+ "shillelagh.backends.apsw.dialects.base.get_adapter_for_table_name"
)
adapter = get_adapter_for_table_name()
adapter._spreadsheet_id = 1