You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by bo...@apache.org on 2023/07/10 18:38:10 UTC
[streampipes] branch dev updated: #1374: Convenient columns query parameter for data lake measure (#1737)
This is an automated email from the ASF dual-hosted git repository.
bossenti pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new bac9e9a32 #1374: Convenient columns query parameter for data lake measure (#1737)
bac9e9a32 is described below
commit bac9e9a321534eedd52888f873508f1ad23d4fa7
Author: Joe Biggins <39...@users.noreply.github.com>
AuthorDate: Mon Jul 10 13:38:04 2023 -0500
#1374: Convenient columns query parameter for data lake measure (#1737)
---
.../streampipes/endpoint/api/data_lake_measure.py | 37 ++++++-
.../tests/endpoint/test_data_lake_measure.py | 110 +++++++++++----------
2 files changed, 94 insertions(+), 53 deletions(-)
diff --git a/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py b/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py
index 1c494895a..cb9497a9c 100644
--- a/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py
+++ b/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py
@@ -20,7 +20,7 @@ Specific implementation of the StreamPipes API's data lake measure endpoints.
This endpoint allows to consume data stored in StreamPipes' data lake.
"""
from datetime import datetime
-from typing import Any, Dict, Literal, Optional, Tuple, Type
+from typing import Any, Dict, List, Literal, Optional, Tuple, Type
from pydantic import BaseModel, Extra, Field, StrictInt, ValidationError, validator
from streampipes.endpoint.endpoint import APIEndpoint
@@ -49,7 +49,7 @@ class MeasurementGetQueryConfig(BaseModel):
Attributes
----------
- columns: Optional[str]
+ columns: Optional[List[str]]
A comma separated list of column names (e.g., `time,value`)<br>
If provided, the returned data only consists of the given columns.<br>
Please be aware that the column `time` as an index is always included.
@@ -87,6 +87,39 @@ class MeasurementGetQueryConfig(BaseModel):
page_no: Optional[int] = Field(alias="page", ge=1)
start_date: Optional[StrictInt] = Field(alias="startDate")
+ @validator("columns", pre=True)
+ @classmethod
+ def _convert_to_comma_separated_string(cls, value: Optional[List[str]]) -> Optional[str]:
+ """Pydantic validator to convert a list to a comma separated string.
+ This is necessary for the StreamPipes API.
+
+ Parameters
+ ----------
+ value: Any
+ The value to be converted to a comma separated string
+
+ Raises
+ ------
+ StreamPipesQueryValidationError
+ In case the provided value is not a list
+
+ Returns
+ -------
+ comma_separated_string: Optional[str]
+ The provided value converted to a comma separated string
+ """
+ if value is None:
+ return value
+ if not isinstance(value, list):
+ raise StreamPipesQueryValidationError(
+ f"The provided value for either `columns`" f"is not a list: '{value}'."
+ )
+ if len(value) == 0:
+ raise StreamPipesQueryValidationError(
+ f"The provided value for either `columns`" f"is an empty list: '{value}'."
+ )
+ return ",".join(value)
+
@validator("end_date", "start_date", pre=True)
@classmethod
def _convert_datetime(cls, dt: datetime) -> int:
diff --git a/streampipes-client-python/tests/endpoint/test_data_lake_measure.py b/streampipes-client-python/tests/endpoint/test_data_lake_measure.py
index 4ee22a607..4ef667539 100644
--- a/streampipes-client-python/tests/endpoint/test_data_lake_measure.py
+++ b/streampipes-client-python/tests/endpoint/test_data_lake_measure.py
@@ -17,11 +17,14 @@
from datetime import datetime
from unittest import TestCase
-from streampipes.endpoint.api.data_lake_measure import DataLakeMeasureEndpoint, StreamPipesQueryValidationError
+from streampipes.endpoint.api.data_lake_measure import (
+ DataLakeMeasureEndpoint,
+ StreamPipesQueryValidationError,
+)
-class TestMeasurementGetQueryConfig(TestCase):
+class TestMeasurementGetQueryConfig(TestCase):
def test_default(self):
config_dict = {}
measurement_config = DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
@@ -30,9 +33,7 @@ class TestMeasurementGetQueryConfig(TestCase):
self.assertEqual("?limit=1000", result)
def test_additional_param_given(self):
- config_dict = {
- "columns": "time,value_25"
- }
+ config_dict = {"columns": ["time", "value_25"]}
measurement_config = DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
result = measurement_config.build_query_string()
@@ -40,17 +41,13 @@ class TestMeasurementGetQueryConfig(TestCase):
self.assertEqual("?columns=time,value_25&limit=1000", result)
def test_extra_param(self):
- config_dict = {
- "foo": "bar"
- }
+ config_dict = {"foo": "bar"}
with self.assertRaises(StreamPipesQueryValidationError):
DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
def test_alias_as_query_param(self):
- config_dict = {
- "page_no": 5
- }
+ config_dict = {"page_no": 5}
measurement_config = DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
result = measurement_config.build_query_string()
@@ -60,10 +57,7 @@ class TestMeasurementGetQueryConfig(TestCase):
def test_datetime_validation(self):
now = datetime.utcnow()
- config_dict = {
- "start_date": now,
- "end_date": now
- }
+ config_dict = {"start_date": now, "end_date": now}
measurement_config = DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
result = measurement_config.build_query_string()
@@ -73,49 +67,66 @@ class TestMeasurementGetQueryConfig(TestCase):
self.assertEqual(expected, result)
def test_datetime_validation_no_datetime(self):
- config_dict = {
- "start_date": "test"
- }
+ config_dict = {"start_date": "test"}
with self.assertRaises(StreamPipesQueryValidationError):
DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
def test_columns_validation(self):
- config_dict_one_col = {
- "columns": "col1"
- }
- config_dict_mul_col = {
- "columns": "col1,col2,col3"
- }
- config_dict_semicolon = {
- "columns": "col1;col2"
- }
- config_dict_whitespace_ending = {
- "columns": "col1 "
- }
-
- self.assertEqual("?columns=col1&limit=1000", DataLakeMeasureEndpoint._validate_query_params(
- query_params=config_dict_one_col).build_query_string())
- self.assertEqual("?columns=col1,col2,col3&limit=1000", DataLakeMeasureEndpoint._validate_query_params(
- query_params=config_dict_mul_col).build_query_string())
+ # Column parameter validation tests:
+
+ # 1. Valid column parameter values
+ config_dict_one_col = {"columns": ["col1"]}
+ config_dict_mul_col = {"columns": ["col1", "col2", "col3"]}
+ config_dict_default_value = {"columns": None}
+
+ # 2. Invalid column parameter values
+ config_dict_whitespace_ending = {"columns": ["col1 "]}
+ config_dict_semicolon = {"columns": "col1;col2"}
+ config_dict_empty_list = {"columns": []}
+ config_dict_string = {"columns": "colstring"}
+ config_dict_integer = {"columns": 1}
+ config_dict_tuple = {"columns": ("col1", "col2")}
+ config_dict_list_with_non_strings = {"columns": ["col1", "col2", 12, ["co3"]]}
+ config_dict_list_with_elems_containing_invalid_chars = {"columns": ["col1", "col2", "col,3", "col_4", "col 5"]}
+
+ self.assertEqual(
+ "?columns=col1&limit=1000",
+ DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_one_col).build_query_string(),
+ )
+ self.assertEqual(
+ "?columns=col1,col2,col3&limit=1000",
+ DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_mul_col).build_query_string(),
+ )
+ self.assertEqual(
+ "?limit=1000",
+ DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_default_value).build_query_string(),
+ )
+ with self.assertRaises(StreamPipesQueryValidationError):
+ DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_whitespace_ending)
+ with self.assertRaises(StreamPipesQueryValidationError):
+ DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_empty_list)
with self.assertRaises(StreamPipesQueryValidationError):
DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_semicolon)
with self.assertRaises(StreamPipesQueryValidationError):
- DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_whitespace_ending)
+ DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_string)
+ with self.assertRaises(StreamPipesQueryValidationError):
+ DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_integer)
+ with self.assertRaises(StreamPipesQueryValidationError):
+ DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_tuple)
+ with self.assertRaises(StreamPipesQueryValidationError):
+ DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_list_with_non_strings)
+ with self.assertRaises(StreamPipesQueryValidationError):
+ DataLakeMeasureEndpoint._validate_query_params(
+ query_params=config_dict_list_with_elems_containing_invalid_chars
+ )
def test_minium_parameter_values(self):
- config_dict_happy_path = {
- "limit": 15,
- "page_no": 3
- }
+ config_dict_happy_path = {"limit": 15, "page_no": 3}
- config_dict_limit_too_low = {
- "limit": 0
- }
+ config_dict_limit_too_low = {"limit": 0}
- config_dict_page_no_too_low = {
- "page_no": -2
- }
+ config_dict_page_no_too_low = {"page_no": -2}
measurement_config_happy = DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_happy_path)
result_happy = measurement_config_happy.build_query_string()
@@ -129,10 +140,7 @@ class TestMeasurementGetQueryConfig(TestCase):
DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_page_no_too_low)
def test_literal_validation(self):
-
- config_invalid_order = {
- "order": "UP"
- }
+ config_invalid_order = {"order": "UP"}
with self.assertRaises(StreamPipesQueryValidationError):
- DataLakeMeasureEndpoint._validate_query_params(query_params=config_invalid_order)
\ No newline at end of file
+ DataLakeMeasureEndpoint._validate_query_params(query_params=config_invalid_order)