You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by bo...@apache.org on 2023/07/10 18:38:10 UTC

[streampipes] branch dev updated: #1374: Convenient columns query parameter for data lake measure (#1737)

This is an automated email from the ASF dual-hosted git repository.

bossenti pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git


The following commit(s) were added to refs/heads/dev by this push:
     new bac9e9a32 #1374: Convenient columns query parameter for data lake measure (#1737)
bac9e9a32 is described below

commit bac9e9a321534eedd52888f873508f1ad23d4fa7
Author: Joe Biggins <39...@users.noreply.github.com>
AuthorDate: Mon Jul 10 13:38:04 2023 -0500

    #1374: Convenient columns query parameter for data lake measure (#1737)
---
 .../streampipes/endpoint/api/data_lake_measure.py  |  37 ++++++-
 .../tests/endpoint/test_data_lake_measure.py       | 110 +++++++++++----------
 2 files changed, 94 insertions(+), 53 deletions(-)

diff --git a/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py b/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py
index 1c494895a..cb9497a9c 100644
--- a/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py
+++ b/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py
@@ -20,7 +20,7 @@ Specific implementation of the StreamPipes API's data lake measure endpoints.
 This endpoint allows to consume data stored in StreamPipes' data lake.
 """
 from datetime import datetime
-from typing import Any, Dict, Literal, Optional, Tuple, Type
+from typing import Any, Dict, List, Literal, Optional, Tuple, Type
 
 from pydantic import BaseModel, Extra, Field, StrictInt, ValidationError, validator
 from streampipes.endpoint.endpoint import APIEndpoint
@@ -49,7 +49,7 @@ class MeasurementGetQueryConfig(BaseModel):
 
     Attributes
     ----------
-    columns: Optional[str]
+    columns: Optional[List[str]]
         A comma separated list of column names (e.g., `time,value`)<br>
         If provided, the returned data only consists of the given columns.<br>
         Please be aware that the column `time` as an index is always included.
@@ -87,6 +87,39 @@ class MeasurementGetQueryConfig(BaseModel):
     page_no: Optional[int] = Field(alias="page", ge=1)
     start_date: Optional[StrictInt] = Field(alias="startDate")
 
+    @validator("columns", pre=True)
+    @classmethod
+    def _convert_to_comma_separated_string(cls, value: Optional[List[str]]) -> Optional[str]:
+        """Pydantic validator to convert a list to a comma separated string.
+        This is necessary for the StreamPipes API.
+
+        Parameters
+        ----------
+        value: Any
+            The value to be converted to a comma separated string
+
+        Raises
+        ------
+        StreamPipesQueryValidationError
+            In case the provided value is not a list
+
+        Returns
+        -------
+        comma_separated_string: Optional[str]
+            The provided value converted to a comma separated string
+        """
+        if value is None:
+            return value
+        if not isinstance(value, list):
+            raise StreamPipesQueryValidationError(
+                f"The provided value for either `columns`" f"is not a list: '{value}'."
+            )
+        if len(value) == 0:
+            raise StreamPipesQueryValidationError(
+                f"The provided value for either `columns`" f"is an empty list: '{value}'."
+            )
+        return ",".join(value)
+
     @validator("end_date", "start_date", pre=True)
     @classmethod
     def _convert_datetime(cls, dt: datetime) -> int:
diff --git a/streampipes-client-python/tests/endpoint/test_data_lake_measure.py b/streampipes-client-python/tests/endpoint/test_data_lake_measure.py
index 4ee22a607..4ef667539 100644
--- a/streampipes-client-python/tests/endpoint/test_data_lake_measure.py
+++ b/streampipes-client-python/tests/endpoint/test_data_lake_measure.py
@@ -17,11 +17,14 @@
 
 from datetime import datetime
 from unittest import TestCase
-from streampipes.endpoint.api.data_lake_measure import DataLakeMeasureEndpoint, StreamPipesQueryValidationError
 
+from streampipes.endpoint.api.data_lake_measure import (
+    DataLakeMeasureEndpoint,
+    StreamPipesQueryValidationError,
+)
 
-class TestMeasurementGetQueryConfig(TestCase):
 
+class TestMeasurementGetQueryConfig(TestCase):
     def test_default(self):
         config_dict = {}
         measurement_config = DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
@@ -30,9 +33,7 @@ class TestMeasurementGetQueryConfig(TestCase):
         self.assertEqual("?limit=1000", result)
 
     def test_additional_param_given(self):
-        config_dict = {
-            "columns": "time,value_25"
-        }
+        config_dict = {"columns": ["time", "value_25"]}
 
         measurement_config = DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
         result = measurement_config.build_query_string()
@@ -40,17 +41,13 @@ class TestMeasurementGetQueryConfig(TestCase):
         self.assertEqual("?columns=time,value_25&limit=1000", result)
 
     def test_extra_param(self):
-        config_dict = {
-            "foo": "bar"
-        }
+        config_dict = {"foo": "bar"}
 
         with self.assertRaises(StreamPipesQueryValidationError):
             DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
 
     def test_alias_as_query_param(self):
-        config_dict = {
-            "page_no": 5
-        }
+        config_dict = {"page_no": 5}
 
         measurement_config = DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
         result = measurement_config.build_query_string()
@@ -60,10 +57,7 @@ class TestMeasurementGetQueryConfig(TestCase):
     def test_datetime_validation(self):
         now = datetime.utcnow()
 
-        config_dict = {
-            "start_date": now,
-            "end_date": now
-        }
+        config_dict = {"start_date": now, "end_date": now}
         measurement_config = DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
         result = measurement_config.build_query_string()
 
@@ -73,49 +67,66 @@ class TestMeasurementGetQueryConfig(TestCase):
         self.assertEqual(expected, result)
 
     def test_datetime_validation_no_datetime(self):
-        config_dict = {
-            "start_date": "test"
-        }
+        config_dict = {"start_date": "test"}
 
         with self.assertRaises(StreamPipesQueryValidationError):
             DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict)
 
     def test_columns_validation(self):
-        config_dict_one_col = {
-            "columns": "col1"
-        }
-        config_dict_mul_col = {
-            "columns": "col1,col2,col3"
-        }
-        config_dict_semicolon = {
-            "columns": "col1;col2"
-        }
-        config_dict_whitespace_ending = {
-            "columns": "col1 "
-        }
-
-        self.assertEqual("?columns=col1&limit=1000", DataLakeMeasureEndpoint._validate_query_params(
-            query_params=config_dict_one_col).build_query_string())
-        self.assertEqual("?columns=col1,col2,col3&limit=1000", DataLakeMeasureEndpoint._validate_query_params(
-            query_params=config_dict_mul_col).build_query_string())
+        # Column parameter validation tests:
+
+        # 1. Valid column parameter values
+        config_dict_one_col = {"columns": ["col1"]}
+        config_dict_mul_col = {"columns": ["col1", "col2", "col3"]}
+        config_dict_default_value = {"columns": None}
+
+        # 2. Invalid column parameter values
+        config_dict_whitespace_ending = {"columns": ["col1 "]}
+        config_dict_semicolon = {"columns": "col1;col2"}
+        config_dict_empty_list = {"columns": []}
+        config_dict_string = {"columns": "colstring"}
+        config_dict_integer = {"columns": 1}
+        config_dict_tuple = {"columns": ("col1", "col2")}
+        config_dict_list_with_non_strings = {"columns": ["col1", "col2", 12, ["co3"]]}
+        config_dict_list_with_elems_containing_invalid_chars = {"columns": ["col1", "col2", "col,3", "col_4", "col 5"]}
+
+        self.assertEqual(
+            "?columns=col1&limit=1000",
+            DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_one_col).build_query_string(),
+        )
+        self.assertEqual(
+            "?columns=col1,col2,col3&limit=1000",
+            DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_mul_col).build_query_string(),
+        )
+        self.assertEqual(
+            "?limit=1000",
+            DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_default_value).build_query_string(),
+        )
+        with self.assertRaises(StreamPipesQueryValidationError):
+            DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_whitespace_ending)
+        with self.assertRaises(StreamPipesQueryValidationError):
+            DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_empty_list)
         with self.assertRaises(StreamPipesQueryValidationError):
             DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_semicolon)
         with self.assertRaises(StreamPipesQueryValidationError):
-            DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_whitespace_ending)
+            DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_string)
+        with self.assertRaises(StreamPipesQueryValidationError):
+            DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_integer)
+        with self.assertRaises(StreamPipesQueryValidationError):
+            DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_tuple)
+        with self.assertRaises(StreamPipesQueryValidationError):
+            DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_list_with_non_strings)
+        with self.assertRaises(StreamPipesQueryValidationError):
+            DataLakeMeasureEndpoint._validate_query_params(
+                query_params=config_dict_list_with_elems_containing_invalid_chars
+            )
 
     def test_minium_parameter_values(self):
-        config_dict_happy_path = {
-            "limit": 15,
-            "page_no": 3
-        }
+        config_dict_happy_path = {"limit": 15, "page_no": 3}
 
-        config_dict_limit_too_low = {
-            "limit": 0
-        }
+        config_dict_limit_too_low = {"limit": 0}
 
-        config_dict_page_no_too_low = {
-            "page_no": -2
-        }
+        config_dict_page_no_too_low = {"page_no": -2}
 
         measurement_config_happy = DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_happy_path)
         result_happy = measurement_config_happy.build_query_string()
@@ -129,10 +140,7 @@ class TestMeasurementGetQueryConfig(TestCase):
             DataLakeMeasureEndpoint._validate_query_params(query_params=config_dict_page_no_too_low)
 
     def test_literal_validation(self):
-
-        config_invalid_order = {
-            "order": "UP"
-        }
+        config_invalid_order = {"order": "UP"}
 
         with self.assertRaises(StreamPipesQueryValidationError):
-            DataLakeMeasureEndpoint._validate_query_params(query_params=config_invalid_order)
\ No newline at end of file
+            DataLakeMeasureEndpoint._validate_query_params(query_params=config_invalid_order)