You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by bo...@apache.org on 2022/11/12 15:01:33 UTC
[incubator-streampipes] 18/18: tmp
This is an automated email from the ASF dual-hosted git repository.
bossenti pushed a commit to branch feature/STREAMPIPES-607
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit f4f1b0410645d827072e044dc76f5b7bab998bbb
Author: bossenti <bo...@posteo.de>
AuthorDate: Sat Nov 12 16:00:55 2022 +0100
tmp
---
.../streampipes_client/__init__.py | 10 +-
.../streampipes_client/client/__init__.py | 2 +-
.../streampipes_client/client/client.py | 10 +-
.../client/credential_provider.py | 111 +++++++++++++++
.../client/endpoint/data_lake_measure.py | 9 +-
.../streampipes_client/client/endpoint/endpoint.py | 20 +--
.../streampipes_client/model/common.py | 58 ++++----
.../streampipes_client/model/container/__init__.py | 13 +-
.../model/container/data_lake_measures.py | 59 +++++++-
.../model/container/model_container.py | 60 --------
.../model/container/resource_container.py | 151 +++++++++++++++++++++
.../streampipes_client/model/element/__init__.py | 18 ---
.../streampipes_client/model/exception.py | 86 ++++++++++++
.../{element/element.py => resource/__init__.py} | 11 +-
.../{element => resource}/data_lake_measure.py | 18 ++-
.../credentials.py => model/resource/resource.py} | 40 +++---
16 files changed, 502 insertions(+), 174 deletions(-)
diff --git a/streampipes-client-python/streampipes_client/__init__.py b/streampipes-client-python/streampipes_client/__init__.py
index 285906ad1..95500c32f 100644
--- a/streampipes-client-python/streampipes_client/__init__.py
+++ b/streampipes-client-python/streampipes_client/__init__.py
@@ -17,12 +17,4 @@
"""
This library provides a handy Python-based client to interact with Apache StreamPipes.
-"""
-
-from .client import StreamPipesClient, StreamPipesClientConfig, StreamPipesApiKeyCredentials
-
-__all = [
- "StreamPipesApiKeyCredentials"
- "StreamPipesClient",
- "StreamPipesClientConfig",
-]
+"""
\ No newline at end of file
diff --git a/streampipes-client-python/streampipes_client/client/__init__.py b/streampipes-client-python/streampipes_client/client/__init__.py
index 98d1d0e37..35f6c8530 100644
--- a/streampipes-client-python/streampipes_client/client/__init__.py
+++ b/streampipes-client-python/streampipes_client/client/__init__.py
@@ -20,7 +20,7 @@ This module contains everything about the client itself, the central point of in
"""
from .client import StreamPipesClient, StreamPipesClientConfig
-from .credentials import StreamPipesApiKeyCredentials
+from .credential_provider import StreamPipesApiKeyCredentials
__all__ = [
"StreamPipesApiKeyCredentials",
diff --git a/streampipes-client-python/streampipes_client/client/client.py b/streampipes-client-python/streampipes_client/client/client.py
index 9e7e2b471..91ae647bd 100644
--- a/streampipes-client-python/streampipes_client/client/client.py
+++ b/streampipes-client-python/streampipes_client/client/client.py
@@ -21,13 +21,15 @@ The client is designed as the central point of interaction with the StreamPipes
provides all functionalities to communicate with the API.
"""
+from __future__ import annotations
+
import logging
import sys
from dataclasses import dataclass
from typing import Dict, Optional
import requests
-from streampipes_client.client.credentials import CredentialProvider
+from streampipes_client.client.credential_provider import CredentialProvider
from streampipes_client.client.endpoint import DataLakeMeasureEndpoint
logger = logging.getLogger(__name__)
@@ -80,7 +82,7 @@ class StreamPipesClient:
Examples
--------
- >>> from streampipes_client import StreamPipesClient, StreamPipesClientConfig, StreamPipesApiKeyCredentials
+ >>> from streampipes_client.client import StreamPipesClient, StreamPipesClientConfig, StreamPipesApiKeyCredentials
>>> client_config = StreamPipesClientConfig(
... credential_provider=StreamPipesApiKeyCredentials(
@@ -144,7 +146,7 @@ class StreamPipesClient:
cls,
client_config: StreamPipesClientConfig,
logging_level: int = logging.INFO,
- ):
+ ) -> StreamPipesClient:
"""Returns an instance of the `StreamPipesPythonClient`.
Provides consistency to the Java client.
@@ -169,7 +171,7 @@ class StreamPipesClient:
Returns
-------
- Dictionary with header information as string key-value pairs
+ Dictionary with header information as string key-value pairs.
"""
# create HTTP headers from credential provider and add additional headers needed
diff --git a/streampipes-client-python/streampipes_client/client/credential_provider.py b/streampipes-client-python/streampipes_client/client/credential_provider.py
new file mode 100644
index 000000000..dd99dd6ba
--- /dev/null
+++ b/streampipes-client-python/streampipes_client/client/credential_provider.py
@@ -0,0 +1,111 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Implementation of credential providers.
+A credential provider supplies the specified sort of credentials in the appropriate HTTP header format.
+The headers are then used by the client to connect to StreamPipes.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Dict, Optional
+
+__all__ = [
+ "StreamPipesApiKeyCredentials",
+]
+
+
+class CredentialProvider(ABC):
+ """Abstract implementation of a credential provider.
+ Must be inherited by all credential providers.
+ """
+
+ def make_headers(self, http_headers: Optional[Dict[str, str]] = None) -> Dict[str, str]:
+ """Creates the HTTP headers for the specific credential provider.
+ Concrete authentication headers must be defined in the implementation of a credential provider.
+
+ Parameters
+ ----------
+ http_headers: Optional[Dict[str, str]]
+ Additional HTTP headers the generated headers are extended by.
+
+ Returns
+ -------
+ Dictionary with header information as string key-value pairs.
+
+ """
+ if http_headers is None:
+ http_headers = {}
+
+ http_headers.update(self._authentication_headers)
+
+ return http_headers
+
+ @property
+ @abstractmethod
+ def _authentication_headers(self) -> Dict[str, str]:
+ """Provides the HTTP headers used for the authentication with the concrete `CredentialProvider`.
+
+ Returns
+ -------
+ Dictionary with authentication headers as string key-value pairs.
+
+ """
+ raise NotImplementedError
+
+
+class StreamPipesApiKeyCredentials(CredentialProvider):
+ """A Credential provider that allows authentication via a StreamPipes API Token.
+ This token can be generated via the StreamPipes UI (see how in the project's README).
+
+ Parameters
+ ----------
+ username: str
+ The username to which the API token is granted, e.g., `demo-user@streampipes.apche.org`.
+ api_key: str
+ The StreamPipes API key as it is displayed in the UI.
+
+ Examples
+ --------
+ see `StreamPipesClient`
+
+ References
+ ----------
+ [^1]: [StreamPipes Python Client README](https://github.com/apache/incubator-streampipes/blob/dev/streampipes-client-python/README.md#%EF%B8%8F-quickstart)
+ """
+
+ def __init__(
+ self,
+ username: str,
+ api_key: str,
+ ):
+ self.username = username
+ self.api_key = api_key
+
+ @property
+ def _authentication_headers(self) -> Dict[str, str]:
+ """Provides the HTTP headers used for the authentication with the API token.
+
+ Returns
+ -------
+ Dictionary with authentication headers as string key-value pairs.
+
+ """
+ return {
+ "X-API-User": self.username,
+ "X-API-Key": self.api_key,
+ }
diff --git a/streampipes-client-python/streampipes_client/client/endpoint/data_lake_measure.py b/streampipes-client-python/streampipes_client/client/endpoint/data_lake_measure.py
index a46338165..c16d1774a 100644
--- a/streampipes-client-python/streampipes_client/client/endpoint/data_lake_measure.py
+++ b/streampipes-client-python/streampipes_client/client/endpoint/data_lake_measure.py
@@ -19,7 +19,10 @@
Specific implementation of the StreamPipes API's data lake measure endpoints.
This endpoint allows to consume data stored in StreamPipes' data lake
"""
+from typing import Tuple, Type
+
from client.endpoint.endpoint import APIEndpoint
+from model.container import ResourceContainer
from streampipes_client.model.container.data_lake_measures import DataLakeMeasures
__all__ = [
@@ -45,7 +48,7 @@ class DataLakeMeasureEndpoint(APIEndpoint):
Examples
--------
- >>> from streampipes_client import StreamPipesClient, StreamPipesClientConfig, StreamPipesApiKeyCredentials
+ >>> from streampipes_client.client import StreamPipesClient, StreamPipesClientConfig, StreamPipesApiKeyCredentials
>>> client_config = StreamPipesClientConfig(
... credential_provider=StreamPipesApiKeyCredentials(username="test-user", api_key="api-key"),
@@ -63,7 +66,7 @@ class DataLakeMeasureEndpoint(APIEndpoint):
"""
@property
- def _container_cls(self):
+ def _container_cls(self) -> Type[ResourceContainer]:
"""Defines the model container class the endpoint refers to.
@@ -74,7 +77,7 @@ class DataLakeMeasureEndpoint(APIEndpoint):
return DataLakeMeasures
@property
- def _relative_api_path(self):
+ def _relative_api_path(self) -> Tuple[str, ...]:
"""Defines the relative api path to the DataLakeMeasurement endpoint.
Each path within the URL is defined as an own string.
diff --git a/streampipes-client-python/streampipes_client/client/endpoint/endpoint.py b/streampipes-client-python/streampipes_client/client/endpoint/endpoint.py
index 7f349b386..f897d6bcf 100644
--- a/streampipes-client-python/streampipes_client/client/endpoint/endpoint.py
+++ b/streampipes-client-python/streampipes_client/client/endpoint/endpoint.py
@@ -28,8 +28,8 @@ from typing import Callable, Tuple, Type, Union
from requests import Response
from requests.exceptions import BaseHTTPError, HTTPError
-from streampipes_client.model.container import ModelContainer
-from streampipes_client.model.element import Element
+from streampipes_client.model.container import ResourceContainer
+from streampipes_client.model.resource import Resource
__all__ = [
"APIEndpoint",
@@ -74,7 +74,7 @@ class APIEndpoint(ABC):
@property
@abstractmethod
- def _container_cls(cls) -> Type[ModelContainer]:
+ def _container_cls(self) -> Type[ResourceContainer]:
"""Defines the model container class the endpoint refers to.
This model container class corresponds to the Python data model,
which handles multiple resources returned from the endpoint.
@@ -82,13 +82,13 @@ class APIEndpoint(ABC):
Returns
-------
The corresponding container class from the data model,
- needs to a subclass of `model.ModelContainer`.
+ needs to a subclass of `model.container.ResourceContainer`.
"""
raise NotImplementedError
@property
@abstractmethod
- def _relative_api_path(self) -> Tuple[str]:
+ def _relative_api_path(self) -> Tuple[str, ...]:
"""Defines the relative api path with regard to the StreamPipes API URL.
Each path within the URL is defined as an own string.
@@ -164,23 +164,23 @@ class APIEndpoint(ABC):
"""
return f"{self._parent_client.base_api_path}" f"{'/'.join(api_path for api_path in self._relative_api_path)}"
- def all(self) -> ModelContainer:
+ def all(self) -> ResourceContainer:
"""Get all resources of this endpoint provided by the StreamPipes API.
- Results are provided as an instance of a `model.ModelContainer` that
+ Results are provided as an instance of a `model.container.ResourceContainer` that
allows to handle the returned resources in a comfortable and pythonic way.
Returns
-------
- A model container instance (`model.ModelContainer`) bundling the resources returned.
+ A model container instance (`model.container.ResourceContainer`) bundling the resources returned.
"""
response = self._make_request(
request_method=self._parent_client.request_session.get,
url=self.build_url(),
)
- return self._container_cls().from_json(json_string=response.text)
+ return self._container_cls.from_json(json_string=response.text)
- def get(self, identifier: str) -> Element:
+ def get(self, identifier: str) -> Resource:
"""Queries the specified resource from the API endpoint.
Parameters
diff --git a/streampipes-client-python/streampipes_client/model/common.py b/streampipes-client-python/streampipes_client/model/common.py
index 4c78c906c..940d0def7 100644
--- a/streampipes-client-python/streampipes_client/model/common.py
+++ b/streampipes-client-python/streampipes_client/model/common.py
@@ -15,14 +15,22 @@
# limitations under the License.
#
+"""
+Classes of the StreamPipes data model that are commonly shared.
+"""
+
from typing import List, Optional
-from pydantic import BaseModel, StrictBool, StrictInt, StrictStr, ValidationError
+from pydantic import BaseModel, StrictBool, StrictInt, StrictStr, Field
+__all__ = [
+ "BasicModel",
+ "EventSchema"
+]
-def snake_to_camel_case(snake_case_string: str) -> str:
- """
- Converts a string in snake_case format to camelCase style.
+
+def _snake_to_camel_case(snake_case_string: str) -> str:
+ """Converts a string in snake_case format to camelCase style.
"""
tokens = snake_case_string.split("_")
@@ -34,25 +42,27 @@ class BasicModel(BaseModel):
element_id: Optional[StrictStr]
class Config:
- alias_generator = snake_to_camel_case
-
-
-class EventPropertyQualityDefinition(BasicModel):
- pass
+ alias_generator = _snake_to_camel_case
class EventPropertyQualityRequirement(BasicModel):
- minimum_property_quality: Optional[EventPropertyQualityDefinition]
- maximum_property_quality: Optional[EventPropertyQualityDefinition]
+ """
+ Data model of an `EventPropertyQualityRequirement` in compliance to the StreamPipes Backend.
+ """
+ minimum_property_quality: Optional[BasicModel] = Field(alias="EventPropertyQualityDefinition")
+ maximum_property_quality: Optional[BasicModel] = Field(alias="EventPropertyQualityDefinition")
class EventProperty(BasicModel):
+ """
+ Data model of an `EventProperty` in compliance to the StreamPipes Backend.
+ """
label: StrictStr
description: StrictStr
runtime_name: StrictStr
required: StrictBool
domain_properties: List[StrictStr]
- event_property_qualities: List[EventPropertyQualityDefinition]
+ event_property_qualities: List[BasicModel] = Field(alias="EventPropertyQualityDefinition")
requires_event_property_qualities: List[EventPropertyQualityRequirement]
property_scope: Optional[StrictStr]
index: StrictInt
@@ -63,25 +73,7 @@ class EventProperty(BasicModel):
class EventSchema(BasicModel):
+ """
+ Data model of an `EventSchema` in compliance to the StreamPipes Backend.
+ """
event_properties: List[EventProperty]
-
-
-class StreamPipesDataModelError(RuntimeError):
- def __init__(self, validation_error: ValidationError):
- super().__init__(
- self._generate_error_message(
- model=validation_error.model,
- error_description=validation_error.json(),
- )
- )
-
- @staticmethod
- def _generate_error_message(*, model: BasicModel, error_description: str) -> str:
- return (
- f"\nOops, there seems to be a problem with our internal StreamPipes data model.\n"
- f"This should not occur, but unfortunately did.\n"
- f"Therefore, it would be great if you could report this problem as an issue at github.com/apache/incubator-streampipes.\n"
- f"Please don't forget to include the following information:\n\n"
- f"Affected Model class: {str(model)}\n"
- f"Validation error log: {error_description}"
- )
diff --git a/streampipes-client-python/streampipes_client/model/container/__init__.py b/streampipes-client-python/streampipes_client/model/container/__init__.py
index aa5f2a267..ba22996ea 100644
--- a/streampipes-client-python/streampipes_client/model/container/__init__.py
+++ b/streampipes-client-python/streampipes_client/model/container/__init__.py
@@ -15,5 +15,16 @@
# limitations under the License.
#
-from .model_container import ModelContainer
+"""
+This model contains the implementation of all resource container.
+A ResourceContainer is a collection of `model.resource.Resource`s to which the
+response of the StreamPipes API is parsed.
+"""
+
+from .resource_container import ResourceContainer
from .data_lake_measures import DataLakeMeasures
+
+__all__ = [
+ "DataLakeMeasures",
+ "ResourceContainer",
+]
diff --git a/streampipes-client-python/streampipes_client/model/container/data_lake_measures.py b/streampipes-client-python/streampipes_client/model/container/data_lake_measures.py
index 9f36cf86b..ddd8a5756 100644
--- a/streampipes-client-python/streampipes_client/model/container/data_lake_measures.py
+++ b/streampipes-client-python/streampipes_client/model/container/data_lake_measures.py
@@ -1,12 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Implementation of a resource container for the data lake measures endpoint.
+"""
+
import pandas as pd
-from streampipes_client.model.container import ModelContainer
-from streampipes_client.model.element.data_lake_measure import DataLakeMeasure
+from streampipes_client.model.container import ResourceContainer
+from streampipes_client.model.resource.data_lake_measure import DataLakeMeasure
+
+__all__ = [
+ "DataLakeMeasures",
+]
+
+class DataLakeMeasures(ResourceContainer):
+ """Implementation of the resource container for the data lake measures endpoint.
+ This resource container is a collection of data lake measures returned by the StreamPipes API.
+ It is capable of parsing the response content directly into a list of queried `DataLakeMeasure`.
+ Furthermore, the resource container makes them accessible in a pythonic manner.
+
+ Parameters
+ ----------
+ resources: List[DataLakeMeasure]
+ A list of resources (`model.resource.DataLakeMeasure`) to be contained in the `ResourceContainer`.
+
+ """
-class DataLakeMeasures(ModelContainer):
@classmethod
- def _element_cls(cls):
+ def _resource_cls(cls):
+ """Returns the class of the resource that are bundled.
+
+ Returns
+ -------
+ DataLakeMeasure
+ """
return DataLakeMeasure
def to_pandas(self) -> pd.DataFrame:
- pass
+ """Returns the all data lake measures in one Pandas Dataframe.
+
+ Returns
+ -------
+ pd.DataFrame
+ """
+ raise NotImplementedError
diff --git a/streampipes-client-python/streampipes_client/model/container/model_container.py b/streampipes-client-python/streampipes_client/model/container/model_container.py
deleted file mode 100644
index 596545404..000000000
--- a/streampipes-client-python/streampipes_client/model/container/model_container.py
+++ /dev/null
@@ -1,60 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import json
-from abc import ABC, abstractmethod
-from typing import List, Type
-
-import pandas as pd
-from pydantic import ValidationError
-from streampipes_client.model.common import StreamPipesDataModelError
-from streampipes_client.model.element import Element
-
-
-class ModelContainer(ABC):
- def __init__(self, elements: List[Element]):
- self._elements = elements
-
- def __getitem__(self, position: int) -> Element:
- return self._elements[position]
-
- def __len__(self) -> int:
- return len(self._elements)
-
- @classmethod
- def from_json(cls, json_string: str) -> "ModelContainer":
-
- data = json.loads(json_string)
-
- if not type(data) == list:
- raise RuntimeError
-
- try:
-
- model_container = cls(elements=[cls._element_cls().parse_obj(list_item) for list_item in data])
- except ValidationError as ve:
- raise StreamPipesDataModelError(validation_error=ve)
-
- return model_container
-
- @abstractmethod
- def to_pandas(self) -> pd.DataFrame:
- raise NotImplementedError
-
- @classmethod
- @abstractmethod
- def _element_cls(cls) -> Type[Element]:
- raise NotImplementedError
diff --git a/streampipes-client-python/streampipes_client/model/container/resource_container.py b/streampipes-client-python/streampipes_client/model/container/resource_container.py
new file mode 100644
index 000000000..20a64b264
--- /dev/null
+++ b/streampipes-client-python/streampipes_client/model/container/resource_container.py
@@ -0,0 +1,151 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+General and abstract implementation for a resource container.
+A resource container is a collection of resources returned by the StreamPipes API.
+It is capable of parsing the response content directly into a list of queried resources.
+Furthermore, the resource container makes them accessible in a pythonic manner.
+"""
+
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from typing import List, Type, Dict
+
+import pandas as pd
+from pydantic import ValidationError
+from model.exception import StreamPipesDataModelError, StreamPipesResourceContainerJSONError
+from streampipes_client.model.resource import Resource
+
+__all__ = [
+ "ResourceContainer"
+]
+
+
+class ResourceContainer(ABC):
+ """General and abstract implementation for a resource container.
+ A resource container is a collection of resources returned by the StreamPipes API.
+ It is capable of parsing the response content directly into a list of queried resources.
+ Furthermore, the resource container makes them accessible in a pythonic manner.
+
+ Parameters
+ ----------
+ resources: List[Resource]
+ A list of resources (`model.resource.Resource`) to be contained in the `ResourceContainer`.
+
+ """
+ def __init__(self, resources: List[Resource]):
+ self._resources = resources
+
+ def __getitem__(self, position: int) -> Resource:
+ return self._resources[position]
+
+ def __len__(self) -> int:
+ return len(self._resources)
+
+ def __repr__(self):
+ new_line = '\n'
+ return f"{self.__name__}(resources=[{new_line.join([r.__repr__() for r in self._resources])}])"
+
+ @classmethod
+ @abstractmethod
+ def _resource_cls(cls) -> Type[Resource]:
+ """Returns the class of the resource that are bundled.
+
+ Returns
+ -------
+ model.resource.Resource
+ """
+ raise NotImplementedError
+
+ @classmethod
+ def from_json(cls, json_string: str) -> ResourceContainer:
+ """Creates a `ResourceContainer` from the given JSON string.
+
+ Parameters
+ ----------
+ json_string: str
+ The JSON string returned from the StreamPipes API.
+
+ Returns
+ -------
+ ResourceContainer
+
+ Raises
+ ------
+ StreamPipesDataModelError
+ If a resource cannot be mapped to the corresponding Python data model.
+ StreamPipesResourceContainerJSONError
+ If JSON response cannot be parsed to a `ResourceContainer`.
+ """
+
+ # deserialize JSON string
+ parsed_json = json.loads(json_string)
+
+ # the ResourceContainer expects a list of items
+ # raise an exception if the response does not be a list
+ if not type(parsed_json) == list:
+ raise StreamPipesResourceContainerJSONError(
+ container=cls.__class__,
+ json_string=json_string
+ )
+ try:
+
+ resource_container = cls(resources=[cls._resource_cls().parse_obj(item) for item in parsed_json])
+ except ValidationError as ve:
+ raise StreamPipesDataModelError(validation_error=ve)
+
+ return resource_container
+
+ def to_dicts(self, use_source_names: bool = False) -> List[Dict]:
+ """Returns the contained resources as list of dictionaries.
+
+ Parameters
+ ----------
+ use_source_names: bool
+ Determines whether the field names are named in Python style (=`False`) or
+ as originally named by StreamPipes (=`True`).
+
+ Returns
+ -------
+ List[Dict]]
+ """
+ return [
+ resource.dict(by_alias=use_source_names) for resource in self._resources
+ ]
+
+ def to_json(self) -> str:
+ """Returns the resource container in the StreamPipes JSON representation.
+
+ Returns
+ -------
+ JSON string
+ """
+
+ return json.dumps(self.to_dicts(use_source_names=True))
+
+ @abstractmethod
+ def to_pandas(self) -> pd.DataFrame:
+ """Returns the resource container in representation of a Pandas Dataframe.
+
+ Returns
+ -------
+ pd.DataFrame
+ """
+ raise NotImplementedError
diff --git a/streampipes-client-python/streampipes_client/model/element/__init__.py b/streampipes-client-python/streampipes_client/model/element/__init__.py
deleted file mode 100644
index a8f3b03aa..000000000
--- a/streampipes-client-python/streampipes_client/model/element/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from .element import Element
diff --git a/streampipes-client-python/streampipes_client/model/exception.py b/streampipes-client-python/streampipes_client/model/exception.py
new file mode 100644
index 000000000..00808216b
--- /dev/null
+++ b/streampipes-client-python/streampipes_client/model/exception.py
@@ -0,0 +1,86 @@
+from abc import abstractmethod
+
+from pydantic import ValidationError
+
+from model.common import BasicModel
+
+__all__ = [
+ "StreamPipesDataModelError",
+ "StreamPipesResourceContainerJSONError"
+]
+
+
+class StreamPipesBaseException(Exception):
+ """Basic class for custom StreamPipes exceptions.
+ """
+
+ @staticmethod
+ @abstractmethod
+ def _generate_error_message(**kwargs):
+ raise NotImplementedError
+
+
+class StreamPipesDataModelError(StreamPipesBaseException):
+ """A custom exception to be raised when a validation error occurs
+ during the parsing of StreamPipes API responses.
+
+ Parameters
+ ----------
+ validation_error: ValidationError
+ The validation error thrown by Pydantic during parsing.
+ """
+ def __init__(self,
+ validation_error: ValidationError,
+ ):
+ super().__init__(
+ self._generate_error_message(
+ model=validation_error.model,
+ error_description=validation_error.json(),
+ )
+ )
+
+ @staticmethod
+ def _generate_error_message(model: BasicModel, error_description: str) -> str:
+ return (
+ f"\nOops, there seems to be a problem with our internal StreamPipes data model.\n"
+ f"This should not occur, but unfortunately did.\n"
+ f"Therefore, it would be great if you could report this problem as an issue at github.com/apache/incubator-streampipes.\n"
+ f"Please don't forget to include the following information:\n\n"
+ f"Affected Model class: {str(model)}\n"
+ f"Validation error log: {error_description}"
+ )
+
+
+class StreamPipesResourceContainerJSONError(StreamPipesBaseException):
+ """A custom exception to be raised when the returned JSON string
+ does not suit to the structure of resource container.
+
+ Parameters
+ ----------
+ container: ResourceContainer
+ The class of the resource container where the invalid data structure was detected.
+ json_string: str
+ The JSON string that has been tried to parse.
+ """
+ def __init__(self,
+ container: "ResourceContainer",
+ json_string: str,
+ ):
+ super().__init__(
+ self._generate_error_message(
+ container=container,
+ json_string=json_string,
+ )
+ )
+
+ @staticmethod
+ def _generate_error_message(container: "ResourceContainer", json_string: str):
+ return (
+ f"\nOops, there seems to be a problem when parsing the response of the StreamPipes API."
+ f"This should not occur, but unfortunately did.\n"
+ f"Therefore, it would be great if you could report this problem as an issue at "
+ f"github.com/apache/incubator-streampipes.\n"
+ f"Please don't forget to include the following information:\n\n"
+ f"Affected container class: {str(container)}\n"
+ f"JSON string: {json_string}"
+ )
diff --git a/streampipes-client-python/streampipes_client/model/element/element.py b/streampipes-client-python/streampipes_client/model/resource/__init__.py
similarity index 85%
rename from streampipes-client-python/streampipes_client/model/element/element.py
rename to streampipes-client-python/streampipes_client/model/resource/__init__.py
index d52f261f7..15b2423ce 100644
--- a/streampipes-client-python/streampipes_client/model/element/element.py
+++ b/streampipes-client-python/streampipes_client/model/resource/__init__.py
@@ -14,10 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from abc import ABC
-from streampipes_client.model.common import BasicModel
+from .resource import Resource
+from .data_lake_measure import DataLakeMeasure
-
-class Element(ABC, BasicModel):
- pass
+__all__ = [
+ "DataLakeMeasure",
+ "Resource"
+]
diff --git a/streampipes-client-python/streampipes_client/model/element/data_lake_measure.py b/streampipes-client-python/streampipes_client/model/resource/data_lake_measure.py
similarity index 64%
rename from streampipes-client-python/streampipes_client/model/element/data_lake_measure.py
rename to streampipes-client-python/streampipes_client/model/resource/data_lake_measure.py
index ca5a89fd9..7d5363724 100644
--- a/streampipes-client-python/streampipes_client/model/element/data_lake_measure.py
+++ b/streampipes-client-python/streampipes_client/model/resource/data_lake_measure.py
@@ -18,10 +18,24 @@ from typing import Optional
from pydantic import StrictBool, StrictStr
from streampipes_client.model.common import EventSchema
-from streampipes_client.model.element import Element
+from streampipes_client.model.resource import Resource
+"""
+Implementation of a resource for a data lake measure.
+"""
-class DataLakeMeasure(Element):
+
+__all__ = [
+ "DataLakeMeasure",
+]
+
+class DataLakeMeasure(Resource):
+ """Implementation of a resource for data lake measures.
+ This resource defines the data model used by resource container (`model.container.DataLakeMeasures`).
+ It inherits from Pydantic's BaseModel to get all its superpowers,
+ which are used to parse, validate the API response and to easily switch between
+ the Python representation (both serialized and deserialized) and Java representation (serialized only).
+ """
measure_name: StrictStr
timestamp_field: StrictStr
event_schema: EventSchema
diff --git a/streampipes-client-python/streampipes_client/client/credentials.py b/streampipes-client-python/streampipes_client/model/resource/resource.py
similarity index 51%
rename from streampipes-client-python/streampipes_client/client/credentials.py
rename to streampipes-client-python/streampipes_client/model/resource/resource.py
index de60cb95b..804fb43a1 100644
--- a/streampipes-client-python/streampipes_client/client/credentials.py
+++ b/streampipes-client-python/streampipes_client/model/resource/resource.py
@@ -14,31 +14,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from abc import ABC, abstractmethod
-from typing import Dict, Optional
+"""
+General and abstract implementation for a resource.
+A resource defines the data model that is used by a resource container (`model.container.resourceContainer`).
+"""
+from abc import ABC
-class CredentialProvider(ABC):
- @abstractmethod
- def make_headers(self, http_headers: Optional[Dict[str, str]] = None) -> Dict[str, str]:
- raise NotImplementedError
+from streampipes_client.model.common import BasicModel
+__all__ = [
+ "Resource",
+]
-class StreamPipesApiKeyCredentials(CredentialProvider):
- def __init__(
- self,
- *,
- username: str,
- api_key: str,
- ):
- self.username = username
- self.api_key = api_key
- def make_headers(self, http_headers: Optional[Dict[str, str]] = None) -> Dict[str, str]:
-
- if http_headers is None:
- http_headers = {}
-
- http_headers.update({"X-API-User": self.username, "X-API-Key": self.api_key})
-
- return http_headers
+class Resource(ABC, BasicModel):
+ """General and abstract implementation for a resource.
+ A resource defines the data model used by a resource container (`model.container.resourceContainer`).
+ It inherits from Pydantic's BaseModel to get all its superpowers,
+ which are used to parse, validate the API response and to easily switch between
+ the Python representation (both serialized and deserialized) and Java representation (serialized only).
+ """
+ pass