You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by el...@apache.org on 2023/07/10 23:00:02 UTC
[superset] branch 2.1 updated (c1dea59280 -> 59f12542bb)
This is an automated email from the ASF dual-hosted git repository.
elizabeth pushed a change to branch 2.1
in repository https://gitbox.apache.org/repos/asf/superset.git
omit c1dea59280 chore(metastore-cache): add codec support (#24586)
omit b66f8263be update changelog
new b38306b508 update changelog
new 59f12542bb chore(metastore-cache): add codec support (#24586)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (c1dea59280)
\
N -- N -- N refs/heads/2.1 (59f12542bb)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
[superset] 02/02: chore(metastore-cache): add codec support (#24586)
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elizabeth pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 59f12542bbeb4f89c8c07b1b4bcbd70a0c02eb17
Author: Ville Brofeldt <33...@users.noreply.github.com>
AuthorDate: Tue Jul 4 15:55:22 2023 +0300
chore(metastore-cache): add codec support (#24586)
---
superset/config.py | 25 ++++++++---
superset/extensions/metastore_cache.py | 38 +++++++++++++----
.../extensions/metastore_cache_test.py | 49 +++++++++++++++++++++-
3 files changed, 96 insertions(+), 16 deletions(-)
diff --git a/superset/config.py b/superset/config.py
index af46d489c9..c089416505 100644
--- a/superset/config.py
+++ b/superset/config.py
@@ -62,6 +62,7 @@ from superset.advanced_data_type.plugins.internet_port import internet_port
from superset.advanced_data_type.types import AdvancedDataType
from superset.constants import CHANGE_ME_SECRET_KEY
from superset.jinja_context import BaseTemplateProcessor
+from superset.key_value.types import JsonKeyValueCodec
from superset.stats_logger import DummyStatsLogger
from superset.superset_typing import CacheConfig
from superset.tasks.types import ExecutorType
@@ -686,20 +687,32 @@ CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "NullCache"}
# Cache for datasource metadata and query results
DATA_CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "NullCache"}
-# Cache for dashboard filter state (`CACHE_TYPE` defaults to `SimpleCache` when
-# running in debug mode unless overridden)
+# Cache for dashboard filter state. `CACHE_TYPE` defaults to `SupersetMetastoreCache`
+# that stores the values in the key-value table in the Superset metastore, as it's
+# required for Superset to operate correctly, but can be replaced by any
+# `Flask-Caching` backend.
FILTER_STATE_CACHE_CONFIG: CacheConfig = {
+ "CACHE_TYPE": "SupersetMetastoreCache",
"CACHE_DEFAULT_TIMEOUT": int(timedelta(days=90).total_seconds()),
- # should the timeout be reset when retrieving a cached value
+ # Should the timeout be reset when retrieving a cached value?
"REFRESH_TIMEOUT_ON_RETRIEVAL": True,
+ # The following parameter only applies to `MetastoreCache`:
+ # How should entries be serialized/deserialized?
+ "CODEC": JsonKeyValueCodec(),
}
-# Cache for explore form data state (`CACHE_TYPE` defaults to `SimpleCache` when
-# running in debug mode unless overridden)
+# Cache for explore form data state. `CACHE_TYPE` defaults to `SupersetMetastoreCache`
+# that stores the values in the key-value table in the Superset metastore, as it's
+# required for Superset to operate correctly, but can be replaced by any
+# `Flask-Caching` backend.
EXPLORE_FORM_DATA_CACHE_CONFIG: CacheConfig = {
+ "CACHE_TYPE": "SupersetMetastoreCache",
"CACHE_DEFAULT_TIMEOUT": int(timedelta(days=7).total_seconds()),
- # should the timeout be reset when retrieving a cached value
+ # Should the timeout be reset when retrieving a cached value?
"REFRESH_TIMEOUT_ON_RETRIEVAL": True,
+ # The following parameter only applies to `MetastoreCache`:
+ # How should entries be serialized/deserialized?
+ "CODEC": JsonKeyValueCodec(),
}
# store cache keys by datasource UID (via CacheKey) for custom processing/invalidation
diff --git a/superset/extensions/metastore_cache.py b/superset/extensions/metastore_cache.py
index f69276c908..0dbd42b2b4 100644
--- a/superset/extensions/metastore_cache.py
+++ b/superset/extensions/metastore_cache.py
@@ -14,26 +14,37 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+import logging
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional
from uuid import UUID, uuid3
-from flask import Flask
+from flask import current_app, Flask, has_app_context
from flask_caching import BaseCache
from superset.key_value.exceptions import KeyValueCreateFailedError
-from superset.key_value.types import KeyValueResource, PickleKeyValueCodec
+from superset.key_value.types import (
+ KeyValueCodec,
+ KeyValueResource,
+ PickleKeyValueCodec,
+)
from superset.key_value.utils import get_uuid_namespace
RESOURCE = KeyValueResource.METASTORE_CACHE
-CODEC = PickleKeyValueCodec()
+
+logger = logging.getLogger(__name__)
class SupersetMetastoreCache(BaseCache):
- def __init__(self, namespace: UUID, default_timeout: int = 300) -> None:
+ def __init__(
+ self,
+ namespace: UUID,
+ codec: KeyValueCodec,
+ default_timeout: int = 300,
+ ) -> None:
super().__init__(default_timeout)
self.namespace = namespace
+ self.codec = codec
@classmethod
def factory(
@@ -41,6 +52,17 @@ class SupersetMetastoreCache(BaseCache):
) -> BaseCache:
seed = config.get("CACHE_KEY_PREFIX", "")
kwargs["namespace"] = get_uuid_namespace(seed)
+ codec = config.get("CODEC") or PickleKeyValueCodec()
+ if (
+ has_app_context()
+ and not current_app.debug
+ and isinstance(codec, PickleKeyValueCodec)
+ ):
+ logger.warning(
+ "Using PickleKeyValueCodec with SupersetMetastoreCache may be unsafe, "
+ "use at your own risk."
+ )
+ kwargs["codec"] = codec
return cls(*args, **kwargs)
def get_key(self, key: str) -> UUID:
@@ -69,7 +91,7 @@ class SupersetMetastoreCache(BaseCache):
resource=RESOURCE,
key=self.get_key(key),
value=value,
- codec=CODEC,
+ codec=self.codec,
expires_on=self._get_expiry(timeout),
).run()
return True
@@ -82,7 +104,7 @@ class SupersetMetastoreCache(BaseCache):
CreateKeyValueCommand(
resource=RESOURCE,
value=value,
- codec=CODEC,
+ codec=self.codec,
key=self.get_key(key),
expires_on=self._get_expiry(timeout),
).run()
@@ -98,7 +120,7 @@ class SupersetMetastoreCache(BaseCache):
return GetKeyValueCommand(
resource=RESOURCE,
key=self.get_key(key),
- codec=CODEC,
+ codec=self.codec,
).run()
def has(self, key: str) -> bool:
diff --git a/tests/integration_tests/extensions/metastore_cache_test.py b/tests/integration_tests/extensions/metastore_cache_test.py
index d9e0e9ee26..eeb7279fe1 100644
--- a/tests/integration_tests/extensions/metastore_cache_test.py
+++ b/tests/integration_tests/extensions/metastore_cache_test.py
@@ -16,17 +16,27 @@
# under the License.
from __future__ import annotations
+from contextlib import nullcontext
from datetime import datetime, timedelta
-from typing import TYPE_CHECKING
+from typing import Any, TYPE_CHECKING
from uuid import UUID
import pytest
from flask.ctx import AppContext
from freezegun import freeze_time
+from superset.key_value.exceptions import KeyValueCodecEncodeException
+from superset.key_value.types import (
+ JsonKeyValueCodec,
+ KeyValueCodec,
+ PickleKeyValueCodec,
+)
+
if TYPE_CHECKING:
from superset.extensions.metastore_cache import SupersetMetastoreCache
+NAMESPACE = UUID("ee173d1b-ccf3-40aa-941c-985c15224496")
+
FIRST_KEY = "foo"
FIRST_KEY_INITIAL_VALUE = {"foo": "bar"}
FIRST_KEY_UPDATED_VALUE = "foo"
@@ -40,8 +50,9 @@ def cache() -> SupersetMetastoreCache:
from superset.extensions.metastore_cache import SupersetMetastoreCache
return SupersetMetastoreCache(
- namespace=UUID("ee173d1b-ccf3-40aa-941c-985c15224496"),
+ namespace=NAMESPACE,
default_timeout=600,
+ codec=PickleKeyValueCodec(),
)
@@ -75,3 +86,37 @@ def test_expiry(app_context: AppContext, cache: SupersetMetastoreCache) -> None:
with freeze_time(dttm + delta + timedelta(seconds=1)):
assert cache.has(FIRST_KEY) is False
assert cache.get(FIRST_KEY) is None
+
+
+@pytest.mark.parametrize(
+ "input_,codec,expected_result",
+ [
+ ({"foo": "bar"}, JsonKeyValueCodec(), {"foo": "bar"}),
+ (("foo", "bar"), JsonKeyValueCodec(), ["foo", "bar"]),
+ (complex(1, 1), JsonKeyValueCodec(), KeyValueCodecEncodeException()),
+ ({"foo": "bar"}, PickleKeyValueCodec(), {"foo": "bar"}),
+ (("foo", "bar"), PickleKeyValueCodec(), ("foo", "bar")),
+ (complex(1, 1), PickleKeyValueCodec(), complex(1, 1)),
+ ],
+)
+def test_codec(
+ input_: Any,
+ codec: KeyValueCodec,
+ expected_result: Any,
+ app_context: AppContext,
+) -> None:
+ from superset.extensions.metastore_cache import SupersetMetastoreCache
+
+ cache = SupersetMetastoreCache(
+ namespace=NAMESPACE,
+ default_timeout=600,
+ codec=codec,
+ )
+ cm = (
+ pytest.raises(type(expected_result))
+ if isinstance(expected_result, Exception)
+ else nullcontext()
+ )
+ with cm:
+ cache.set(FIRST_KEY, input_)
+ assert cache.get(FIRST_KEY) == expected_result
[superset] 01/02: update changelog
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elizabeth pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/superset.git
commit b38306b508fbc85e36567c6ae4cbc0e1283e830d
Author: Elizabeth Thompson <es...@gmail.com>
AuthorDate: Fri Jul 7 17:06:51 2023 -0700
update changelog
---
CHANGELOG.md | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6f67b4d5cf..e6fa63ace4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,7 +19,7 @@ under the License.
## Change Log
-- [2.1.1](#211-sun-apr-23-154421-2023--0100)
+- [2.1.1](#211-sun-apr-23-154421-2023-0100)
- [2.1.0](#210-thu-mar-16-211305-2023--0700)
- [2.0.1](#201-fri-nov-4-103402-2022--0400)
- [2.0.0](#200-tue-jun-28-085302-2022--0400)
@@ -38,10 +38,13 @@ under the License.
- [23888](https://github.com/apache/superset/pull/23888) chore(key-value): use json serialization for main resources (@villebro)
**Fixes**
+- [#23723](https://github.com/apache/superset/pull/23723) fix: add enforce URI query params with a specific for MySQL (@dpgaspar)
+- [#23801](https://github.com/apache/superset/pull/23801) fix(plugin-chart-handlebars): Fix TypeError when using handlebars columns raw mode (@fmannhardt)
+- [#23566](https://github.com/apache/superset/pull/23566) fix: Filter values are not updating when dependencies are set (@michael-s-molina)
+- [#23400](https://github.com/apache/superset/pull/23400) fix: Select all issue with "Dynamically search all filter values" in FilterBar (@geido)
- [#23865](https://github.com/apache/superset/pull/23865) fix: Native time range filter in legacy charts (@kgabryje)
- [#24054](https://github.com/apache/superset/pull/24054) fix: handle temporal columns in presto partitions (@giftig)
- [#23882](https://github.com/apache/superset/pull/23882) fix: handle comments in `has_table_query` (@betodealmeida)
-- [#24256](https://github.com/apache/superset/pull/24256) fix: enable strong session protection by default (@dpgaspar)
- [#24137](https://github.com/apache/superset/pull/24137) fix: disable SHOW_STACKTRACE by default (@dpgaspar)
- [#24185](https://github.com/apache/superset/pull/24185) fix: db validate parameters permission (@dpgaspar)
- [#23769](https://github.com/apache/superset/pull/23769) fix: allow db driver distinction on enforced URI params (@dpgaspar)
@@ -58,7 +61,7 @@ under the License.
- [#22851](https://github.com/apache/superset/pull/22851) fix: Validate jinja rendered query (@geido)
**Others**
-- [#23723](https://github.com/apache/superset/pull/23723) chore: add enforce URI query params with a specific for MySQL (@dpgaspar)
+- [#24586](https://github.com/apache/superset/pull/24586) chore(metastore-cache): add codec support
- [#24294](https://github.com/apache/superset/pull/24294) chore: update UPDATING for 2.1.0 (@eschutho)
- [#24056](https://github.com/apache/superset/pull/24056) chore: Remove unnecessary information from response (@geido)