You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by bo...@apache.org on 2022/11/17 16:32:57 UTC
[incubator-streampipes] 02/10: [STREAMPIPES-607] implement basic class structure & first version of the client itself
This is an automated email from the ASF dual-hosted git repository.
bossenti pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 8be545a94402a00a0d6d7b734f6355c3751e2701
Author: bossenti <bo...@posteo.de>
AuthorDate: Sun Nov 13 14:01:32 2022 +0100
[STREAMPIPES-607] implement basic class structure & first version of the client itself
---
.../streampipes_client/{client => }/__init__.py | 2 +-
.../streampipes_client/client/__init__.py | 8 +-
.../streampipes_client/client/client.py | 177 +++++++++++++++++
.../streampipes_client/client/client_config.py | 58 ++++++
.../client/credential_provider.py | 151 +++++++++++++++
.../streampipes_client/endpoint/__init__.py | 7 +-
.../streampipes_client/endpoint/endpoint.py | 197 +++++++++++++++++++
.../streampipes_client/model/__init__.py | 2 +-
.../{client => model/container}/__init__.py | 8 +-
.../model/container/resource_container.py | 212 +++++++++++++++++++++
.../{client => model/resource}/__init__.py | 8 +-
.../streampipes_client/model/resource/resource.py | 43 +++++
streampipes-client-python/tests/__init__.py | 0
.../client/__init__.py | 2 +-
.../tests/client/test_client.py | 69 +++++++
.../tests/client/test_credential_provider.py | 49 +++++
16 files changed, 986 insertions(+), 7 deletions(-)
diff --git a/streampipes-client-python/streampipes_client/client/__init__.py b/streampipes-client-python/streampipes_client/__init__.py
similarity index 99%
copy from streampipes-client-python/streampipes_client/client/__init__.py
copy to streampipes-client-python/streampipes_client/__init__.py
index ecb1860df..cce3acad3 100644
--- a/streampipes-client-python/streampipes_client/client/__init__.py
+++ b/streampipes-client-python/streampipes_client/__init__.py
@@ -13,4 +13,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
\ No newline at end of file
+#
diff --git a/streampipes-client-python/streampipes_client/client/__init__.py b/streampipes-client-python/streampipes_client/client/__init__.py
index ecb1860df..50e4adaa2 100644
--- a/streampipes-client-python/streampipes_client/client/__init__.py
+++ b/streampipes-client-python/streampipes_client/client/__init__.py
@@ -13,4 +13,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
\ No newline at end of file
+#
+
+from .client import StreamPipesClient
+
+__all__ = [
+ "StreamPipesClient",
+]
diff --git a/streampipes-client-python/streampipes_client/client/client.py b/streampipes-client-python/streampipes_client/client/client.py
new file mode 100644
index 000000000..f92a5b1cc
--- /dev/null
+++ b/streampipes-client-python/streampipes_client/client/client.py
@@ -0,0 +1,177 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Implementation of the StreamPipes client.
+The client is designed as the central point of interaction with the StreamPipes API and
+provides all functionalities to communicate with the API.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from typing import Dict, Optional
+
+from requests import Session
+from streampipes_client.client.client_config import StreamPipesClientConfig
+from streampipes_client.endpoint import DataLakeMeasureEndpoint
+
+logger = logging.getLogger(__name__)
+
+
+class StreamPipesClient:
+ """The client to connect to StreamPipes.
+ This is the central point of contact with StreamPipes and
+ provides all the functionalities to interact with it.
+
+ The client provides so-called "endpoints" each of which refers to
+ an endpoint of the StreamPipes API, e.g. `.dataLakeMeasureApi`.
+ An endpoint provides the actual methods to interact with StreamPipes API (see endpoint.endpoint.APIEndpoint).
+
+ Parameters
+ ----------
+ client_config: StreamPipesClientConfig
+ Configures the client to connect properly to the StreamPipes instance.
+ logging_level: Optional[int]
+ Influences the log messages emitted by the `StreamPipesClient`.
+
+ Examples
+ --------
+
+ >>> from streampipes_client.client import StreamPipesClient
+ >>> from streampipes_client.client.client_config import StreamPipesClientConfig
+ >>> from streampipes_client.client.credential_provider import StreamPipesApiKeyCredentials
+
+ >>> client_config = StreamPipesClientConfig(
+ ... credential_provider=StreamPipesApiKeyCredentials(
+ ... username="test-user",
+ ... api_key="api-key"
+ ... ),
+ ... host_address="localhost",
+ ... https_disabled=True
+ ... )
+
+ >>> client = StreamPipesClient.create(client_config=client_config)
+
+ # The above way of instantiating a client instance is intended
+ # to be consistent with the StreamPipes Java client.
+ # If you prefer a more pythonic way, you can simply write:
+ >>> client = StreamPipesClient(client_config=client_config)
+
+ # Interact with an endpoint
+ >>> data_lake_measures = client.dataLakeMeasureApi.all()
+
+ # Inspect returned data as a pandas dataframe
+ >>> data_lake_measures.to_pandas()
+
+ measure_name timestamp_field ... pipeline_is_running num_event_properties
+ 0 test s0::timestamp ... False 2
+ [1 rows x 6 columns]
+
+ """
+
+ def __init__(
+ self,
+ client_config: StreamPipesClientConfig,
+ logging_level: Optional[int] = logging.INFO,
+ ):
+ self.client_config = client_config
+
+ # set up a requests session
+ # this allows to centrally determine the behavior of all requests made
+ self.request_session = Session()
+ self.request_session.headers.update(self.http_headers)
+
+ self._set_up_logging(logging_level=logging_level) # type: ignore
+
+ # provide all available endpoints here
+ # name of the endpoint needs to be consistent with the Java client
+ self.dataLakeMeasureApi = DataLakeMeasureEndpoint(parent_client=self)
+
+ @staticmethod
+ def _set_up_logging(logging_level: int) -> None:
+ """Configures the logging behavior of the `StreamPipesClient`.
+
+ Parameters
+ ----------
+ logging_level: Optional[int]
+ Influences the log messages emitted by the `StreamPipesClient`.
+
+ Returns
+ -------
+ None
+ """
+ logging.basicConfig(
+ level=logging_level,
+ stream=sys.stdout,
+ format="%(asctime)s - %(name)s - [%(levelname)s] - [%(filename)s:%(lineno)d] [%(funcName)s] - %(message)s",
+ )
+
+ logger.info(f"Logging successfully initialized with logging level {logging.getLevelName(logging_level)}.")
+
+ @classmethod
+ def create(
+ cls,
+ client_config: StreamPipesClientConfig,
+ logging_level: int = logging.INFO,
+ ) -> StreamPipesClient:
+ """Returns an instance of the `StreamPipesPythonClient`.
+ Provides consistency to the Java client.
+
+ Parameters
+ ----------
+ client_config: StreamPipesClientConfig
+ Configures the client to connect properly to the StreamPipes instance.
+ logging_level: Optional[int]
+ Influences the log messages emitted by the `StreamPipesClient`.
+
+ Returns
+ -------
+ StreamPipesClient
+ """
+ return cls(client_config=client_config, logging_level=logging_level)
+
+ @property
+ def http_headers(self) -> Dict[str, str]:
+ """Returns the HTTP headers required for all requests.
+ The HTTP headers are composed of the authentication headers supplied by the credential
+ provider and additional required headers (currently this is only the application header).
+
+ Returns
+ -------
+ Dictionary with header information as string key-value pairs.
+ """
+
+ # create HTTP headers from credential provider and add additional headers needed
+ return self.client_config.credential_provider.make_headers(
+ {"Application": "application/json"},
+ )
+
+ @property
+ def base_api_path(self) -> str:
+ """Constructs the basic API URL from the given `client_config`.
+
+ Returns
+ -------
+ str of the basic API URL
+ """
+ return (
+ f"{'http://' if self.client_config.https_disabled else 'https://'}"
+ f"{self.client_config.host_address}:"
+ f"{self.client_config.port}/streampipes-backend/"
+ )
diff --git a/streampipes-client-python/streampipes_client/client/client_config.py b/streampipes-client-python/streampipes_client/client/client_config.py
new file mode 100644
index 000000000..ee98e6bf9
--- /dev/null
+++ b/streampipes-client-python/streampipes_client/client/client_config.py
@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Implementation of a config class for the StreamPipes client.
+"""
+
+
+from dataclasses import dataclass
+from typing import Optional
+
+__all__ = [
+ "StreamPipesClientConfig",
+]
+
+from streampipes_client.client.credential_provider import CredentialProvider
+
+
+@dataclass
+class StreamPipesClientConfig:
+ """Configure the StreamPipes client in accordance to the actual StreamPipes instance to connect to.
+ An instance is provided to the `StreamPipesClient` to configure it properly.
+
+ Parameters
+ ----------
+ credential_provider: CredentialProvider
+ Provides the credentials to authenticate with the StreamPipes API.
+ host_address:
+ Host address of the StreamPipes instance to connect to.
+ Should be provided without the protocol/scheme, e.g. as `localhost` or `streampipes.xyz`.
+ https_disabled: Optional[bool]
+ Determines whether https is used to connect to StreamPipes.
+ port: Optional[int]
+ Specifies the port under which the StreamPipes API is available, e.g., 80.
+
+ Examples
+ --------
+ see `StreamPipesClient`
+ """
+
+ credential_provider: CredentialProvider
+ host_address: str
+ https_disabled: Optional[bool] = False
+ port: Optional[int] = 80
diff --git a/streampipes-client-python/streampipes_client/client/credential_provider.py b/streampipes-client-python/streampipes_client/client/credential_provider.py
new file mode 100644
index 000000000..560f18c53
--- /dev/null
+++ b/streampipes-client-python/streampipes_client/client/credential_provider.py
@@ -0,0 +1,151 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Implementation of credential providers.
+A credential provider supplies the specified sort of credentials in the appropriate HTTP header format.
+The headers are then used by the client to connect to StreamPipes.
+"""
+
+from __future__ import annotations
+
+import os
+from abc import ABC, abstractmethod
+from typing import Dict, Optional
+
+__all__ = [
+ "CredentialProvider",
+ "StreamPipesApiKeyCredentials",
+]
+
+
+class CredentialProvider(ABC):
+ """Abstract implementation of a credential provider.
+ Must be inherited by all credential providers.
+ """
+
+ def make_headers(self, http_headers: Optional[Dict[str, str]] = None) -> Dict[str, str]:
+ """Creates the HTTP headers for the specific credential provider.
+ Concrete authentication headers must be defined in the implementation of a credential provider.
+
+ Parameters
+ ----------
+ http_headers: Optional[Dict[str, str]]
+ Additional HTTP headers the generated headers are extended by.
+
+ Returns
+ -------
+ Dictionary with header information as string key-value pairs.
+
+ """
+ if http_headers is None:
+ http_headers = {}
+
+ http_headers.update(self._authentication_headers)
+
+ return http_headers
+
+ @property
+ @abstractmethod
+ def _authentication_headers(self) -> Dict[str, str]:
+ """Provides the HTTP headers used for the authentication with the concrete `CredentialProvider`.
+
+ Returns
+ -------
+ Dictionary with authentication headers as string key-value pairs.
+
+ """
+ raise NotImplementedError # pragma: no cover
+
+
+class StreamPipesApiKeyCredentials(CredentialProvider):
+ """A Credential provider that allows authentication via a StreamPipes API Token.
+ This token can be generated via the StreamPipes UI (see how in the project's README).
+
+ Parameters
+ ----------
+ username: str
+ The username to which the API token is granted, e.g., `demo-user@streampipes.apche.org`.
+ api_key: str
+ The StreamPipes API key as it is displayed in the UI.
+
+ Examples
+ --------
+ see `StreamPipesClient`
+
+ References
+ ----------
+ [^1]: [StreamPipes Python Client README]
+ (https://github.com/apache/incubator-streampipes/blob/dev/streampipes-client-python/README.md#%EF%B8%8F-quickstart)
+ """
+
+ @classmethod
+ def from_env(cls, username_env: str, api_key_env: str) -> StreamPipesApiKeyCredentials:
+ """Returns an api key provider parameterized via environment variables.
+
+ Parameters
+ ----------
+ username_env: str
+ Name of the environment variable that contains the username
+ api_key_env: str
+ Name of the environment variable that contains the API key
+
+ Returns
+ -------
+ StreamPipesApiKeyCredentials
+
+ Raises
+ ------
+ KeyError
+ If one of the environment variables is not defined
+
+ """
+
+ username = os.getenv(username_env, None)
+ api_key = os.getenv(api_key_env, None)
+
+ if username is None or api_key is None:
+ raise KeyError(
+ f"Ups, the following environment variables have not been found: "
+ f"{'`' + username_env + '`,' if username is None else ''}"
+ f"{'`' + api_key_env +'`' if api_key is None else ''}. " # noqa: W291
+ "Please check them to be properly set."
+ )
+
+ return cls(username=username, api_key=api_key)
+
+ def __init__(
+ self,
+ username: str,
+ api_key: str,
+ ):
+ self.username = username
+ self.api_key = api_key
+
+ @property
+ def _authentication_headers(self) -> Dict[str, str]:
+ """Provides the HTTP headers used for the authentication with the API token.
+
+ Returns
+ -------
+ Dictionary with authentication headers as string key-value pairs.
+
+ """
+ return {
+ "X-API-User": self.username,
+ "X-API-Key": self.api_key,
+ }
diff --git a/streampipes-client-python/streampipes_client/endpoint/__init__.py b/streampipes-client-python/streampipes_client/endpoint/__init__.py
index ecb1860df..93cbf6680 100644
--- a/streampipes-client-python/streampipes_client/endpoint/__init__.py
+++ b/streampipes-client-python/streampipes_client/endpoint/__init__.py
@@ -13,4 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
\ No newline at end of file
+#
+from .data_lake_measure import DataLakeMeasureEndpoint
+
+__all__ = [
+ "DataLakeMeasureEndpoint",
+]
diff --git a/streampipes-client-python/streampipes_client/endpoint/endpoint.py b/streampipes-client-python/streampipes_client/endpoint/endpoint.py
new file mode 100644
index 000000000..0d141fe49
--- /dev/null
+++ b/streampipes-client-python/streampipes_client/endpoint/endpoint.py
@@ -0,0 +1,197 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+General implementation for an endpoint.
+Provided classes and assets are aimed to be used for developing endpoints.
+An endpoint is provides all options to communicate with a central endpoint of the StreamPipes API in a handy way.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from http import HTTPStatus
+from typing import Callable, Tuple, Type
+
+from requests import Response
+from requests.exceptions import HTTPError
+
+__all__ = [
+ "APIEndpoint",
+]
+
+from streampipes_client.model.container.resource_container import ResourceContainer
+from streampipes_client.model.resource.resource import Resource
+
+logger = logging.getLogger(__name__)
+
+# define custom logging messages for some specific HTTP status
+_error_code_to_message = {
+ 401: "\nThe StreamPipes Backend returned an unauthorized error.\n"
+ "Please check your user name and/or password to be correct.",
+ 403: "\nThere seems to be an issue with the access rights of the given user and the resource you queried.\n"
+ "Apparently, this user is not allowed to query the resource.\n"
+ "Please check the user's permissions or contact your StreamPipes admin.",
+ **dict.fromkeys(
+ [404, 405],
+ "\nOops, there seems to be an issue with the Python Client calling the API inappropriately.\n"
+ "This should not happen, but unfortunately did.\n"
+ "If you don't mind, it would be awesome to let us know by creating an issue"
+ " at github.com/apache/incubator-streampipes.\n"
+ "Please paste the following information to the issue description:\n\n",
+ ),
+}
+
+
+class APIEndpoint(ABC):
+ """Abstract implementation of an API endpoint.
+ Serves as template for all endpoints for the StreamPipes API.
+ By design, endpoints are only instantiated within the `__init__` method of the StreamPipesClient.
+
+ Parameters
+ ----------
+ parent_client: StreamPipesClient
+ This parameter expects the instance of the `client.StreamPipesClient` the endpoint is attached to.
+
+ """
+
+ def __init__(self, parent_client: "StreamPipesClient"): # type: ignore # noqa: F821
+ self._parent_client = parent_client
+
+ @property
+ @abstractmethod
+ def _container_cls(self) -> Type[ResourceContainer]:
+ """Defines the model container class the endpoint refers to.
+ This model container class corresponds to the Python data model,
+ which handles multiple resources returned from the endpoint.
+
+ Returns
+ -------
+ The corresponding container class from the data model,
+ needs to a subclass of `model.container.ResourceContainer`.
+ """
+ raise NotImplementedError # pragma: no cover
+
+ @property
+ @abstractmethod
+ def _relative_api_path(self) -> Tuple[str, ...]:
+ """Defines the relative api path with regard to the StreamPipes API URL.
+ Each path within the URL is defined as an own string.
+
+ Returns
+ -------
+ A tuple of strings of which every represents a path value of the endpoint's API URL.
+
+ """
+ raise NotImplementedError # pragma: no cover
+
+ @staticmethod
+ def _make_request(
+ request_method: Callable[..., Response],
+ url: str,
+ ) -> Response:
+ """Helper method to send requests to the StreamPipes API endpoint.
+ Should be used from methods of this class that interacts with the API, e.g. `all()` and `get()`.
+
+ Parameters
+ ----------
+ request_method: Callable[..., Response]
+ The HTTP method with which to submit the request.
+ Must be one of HTTP methods provided by the `requests` library, e.g. `requests.get`.
+ url: str
+ The full URL to which the request should be applied.
+
+ Returns
+ -------
+ An HTTP response, which is of type `requests.Response` and
+ contains both the actual API response and some metadata.
+ Returned only if the request was successful,
+ otherwise it raises an exception (see `Raises`).
+
+ Raises
+ ------
+ requests.exceptions.HTTPError
+ If the HTTP status code of the error is between `400` and `600`.
+ """
+
+ response = request_method(url=url)
+
+ # check if the API request was successful
+ try:
+ response.raise_for_status()
+ except HTTPError as err:
+
+ status_code = err.response.status_code
+
+ # get custom error message based on the returned status code
+ error_message = _error_code_to_message[status_code]
+
+ if status_code in [
+ HTTPStatus.METHOD_NOT_ALLOWED.numerator,
+ HTTPStatus.NOT_FOUND.numerator,
+ ]:
+ error_message += f"url: {err.response.url}\nstatus code: {status_code}"
+
+ logger.debug(f"HTTP error response: {err.response.text}")
+ raise HTTPError(error_message) from err
+
+ else:
+ logger.debug("Successfully retrieved resources from %s.", url)
+ logger.info("Successfully retrieved all resources.")
+
+ return response
+
+ def build_url(self) -> str:
+ """Creates the URL of the API path for the endpoint.
+
+ Returns
+ -------
+ The URL of the Endpoint
+ """
+ return f"{self._parent_client.base_api_path}" f"{'/'.join(api_path for api_path in self._relative_api_path)}"
+
+ def all(self) -> ResourceContainer:
+ """Get all resources of this endpoint provided by the StreamPipes API.
+ Results are provided as an instance of a `model.container.ResourceContainer` that
+ allows to handle the returned resources in a comfortable and pythonic way.
+
+ Returns
+ -------
+ A model container instance (`model.container.ResourceContainer`) bundling the resources returned.
+ """
+
+ response = self._make_request(
+ request_method=self._parent_client.request_session.get,
+ url=self.build_url(),
+ )
+ return self._container_cls.from_json(json_string=response.text)
+
+ def get(self, identifier: str) -> Resource:
+ """Queries the specified resource from the API endpoint.
+
+ Parameters
+ ----------
+ identifier: str
+ The identifier of the resource to be queried.
+
+ Returns
+ -------
+ The specified resource as an instance of the corresponding model class (`model.Element`).
+ """
+ raise NotImplementedError(
+ "We're sorry! This functionality is not yet part of the StreamPipes Python client."
+ "Stay tuned, we will add this shortly."
+ ) # pragma: no cover
diff --git a/streampipes-client-python/streampipes_client/model/__init__.py b/streampipes-client-python/streampipes_client/model/__init__.py
index ecb1860df..cce3acad3 100644
--- a/streampipes-client-python/streampipes_client/model/__init__.py
+++ b/streampipes-client-python/streampipes_client/model/__init__.py
@@ -13,4 +13,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
\ No newline at end of file
+#
diff --git a/streampipes-client-python/streampipes_client/client/__init__.py b/streampipes-client-python/streampipes_client/model/container/__init__.py
similarity index 89%
copy from streampipes-client-python/streampipes_client/client/__init__.py
copy to streampipes-client-python/streampipes_client/model/container/__init__.py
index ecb1860df..612f407b4 100644
--- a/streampipes-client-python/streampipes_client/client/__init__.py
+++ b/streampipes-client-python/streampipes_client/model/container/__init__.py
@@ -13,4 +13,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
\ No newline at end of file
+#
+
+from .data_lake_measures import DataLakeMeasures
+
+__all__ = [
+ "DataLakeMeasures",
+]
diff --git a/streampipes-client-python/streampipes_client/model/container/resource_container.py b/streampipes-client-python/streampipes_client/model/container/resource_container.py
new file mode 100644
index 000000000..90bf2b109
--- /dev/null
+++ b/streampipes-client-python/streampipes_client/model/container/resource_container.py
@@ -0,0 +1,212 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+General and abstract implementation for a resource container.
+A resource container is a collection of resources returned by the StreamPipes API.
+It is capable of parsing the response content directly into a list of queried resources.
+Furthermore, the resource container makes them accessible in a pythonic manner.
+"""
+
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from typing import Dict, List, Type
+
+import pandas as pd
+from pydantic import ValidationError
+
+__all__ = [
+ "ResourceContainer",
+]
+
+from streampipes_client.model.resource.resource import Resource
+
+
+class StreamPipesDataModelError(Exception):
+ """A custom exception to be raised when a validation error occurs
+ during the parsing of StreamPipes API responses.
+
+ Parameters
+ ----------
+ validation_error: ValidationError
+ The validation error thrown by Pydantic during parsing.
+ """
+
+ def __init__(
+ self,
+ validation_error: ValidationError,
+ ):
+ self.validation_error = validation_error
+ super().__init__(self._generate_error_message())
+
+ def _generate_error_message(self) -> str:
+ return (
+ f"\nOops, there seems to be a problem with our internal StreamPipes data model.\n"
+ f"This should not occur, but unfortunately did.\n"
+ f"Therefore, it would be great if you could report this problem as an issue at "
+ f"github.com/apache/incubator-streampipes.\n"
+ f"Please don't forget to include the following information:\n\n"
+ f"Affected Model class: {str(self.validation_error.model)}\n"
+ f"Validation error log: {self.validation_error.json()}"
+ )
+
+
+class StreamPipesResourceContainerJSONError(Exception):
+ """A custom exception to be raised when the returned JSON string
+ does not suit to the structure of resource container.
+
+ Parameters
+ ----------
+ container_name: str
+ The class name of the resource container where the invalid data structure was detected.
+ json_string: str
+ The JSON string that has been tried to parse.
+ """
+
+ def __init__(
+ self,
+ container_name: str, # noqa: F821
+ json_string: str,
+ ):
+ self.container_name = container_name
+ self.json_string = json_string
+ super().__init__(self._generate_error_message())
+
+ def _generate_error_message(self) -> str:
+ return (
+ f"\nOops, there seems to be a problem when parsing the response of the StreamPipes API."
+ f"This should not occur, but unfortunately did.\n"
+ f"Therefore, it would be great if you could report this problem as an issue at "
+ f"github.com/apache/incubator-streampipes.\n"
+ f"Please don't forget to include the following information:\n\n"
+ f"Affected container class: {str(self.container_name)}\n"
+ f"JSON string: {self.json_string}"
+ )
+
+
+class ResourceContainer(ABC):
+ """General and abstract implementation for a resource container.
+ A resource container is a collection of resources returned by the StreamPipes API.
+ It is capable of parsing the response content directly into a list of queried resources.
+ Furthermore, the resource container makes them accessible in a pythonic manner.
+
+ Parameters
+ ----------
+ resources: List[Resource]
+ A list of resources (`model.resource.Resource`) to be contained in the `ResourceContainer`.
+
+ """
+
+ def __init__(self, resources: List[Resource]):
+ self._resources = resources
+
+ def __getitem__(self, position: int) -> Resource:
+ return self._resources[position]
+
+ def __len__(self) -> int:
+ return len(self._resources)
+
+ def __repr__(self):
+ new_line = "\n"
+ return f"{self.__class__.__name__}(resources=[{new_line.join([r.__repr__() for r in self._resources])}])"
+
+ @classmethod
+ @abstractmethod
+ def _resource_cls(cls) -> Type[Resource]:
+ """Returns the class of the resource that are bundled.
+
+ Returns
+ -------
+ model.resource.Resource
+ """
+ raise NotImplementedError # pragma: no cover
+
+ @classmethod
+ def from_json(cls, json_string: str) -> ResourceContainer:
+ """Creates a `ResourceContainer` from the given JSON string.
+
+ Parameters
+ ----------
+ json_string: str
+ The JSON string returned from the StreamPipes API.
+
+ Returns
+ -------
+ ResourceContainer
+
+ Raises
+ ------
+ StreamPipesDataModelError
+ If a resource cannot be mapped to the corresponding Python data model.
+ StreamPipesResourceContainerJSONError
+ If JSON response cannot be parsed to a `ResourceContainer`.
+ """
+
+ # deserialize JSON string
+ parsed_json = json.loads(json_string)
+
+ # the ResourceContainer expects a list of items
+ # raise an exception if the response does not be a list
+ if not type(parsed_json) == list:
+ raise StreamPipesResourceContainerJSONError(container_name=str(cls), json_string=json_string)
+ try:
+
+ resource_container = cls(resources=[cls._resource_cls().parse_obj(item) for item in parsed_json])
+ except ValidationError as ve:
+ raise StreamPipesDataModelError(validation_error=ve)
+
+ return resource_container
+
+ def to_dicts(self, use_source_names: bool = False) -> List[Dict]:
+ """Returns the contained resources as list of dictionaries.
+
+ Parameters
+ ----------
+ use_source_names: bool
+ Determines whether the field names are named in Python style (=`False`) or
+ as originally named by StreamPipes (=`True`).
+
+ Returns
+ -------
+ List[Dict]]
+ """
+ return [resource.dict(by_alias=use_source_names) for resource in self._resources]
+
+ def to_json(self) -> str:
+ """Returns the resource container in the StreamPipes JSON representation.
+
+ Returns
+ -------
+ JSON string
+ """
+
+ return json.dumps(self.to_dicts(use_source_names=True))
+
+ def to_pandas(self) -> pd.DataFrame:
+ """Returns the resource container in representation of a Pandas Dataframe.
+
+ Returns
+ -------
+ pd.DataFrame
+ """
+ return pd.DataFrame.from_records(
+ # ResourceContainer is iterable itself via __get_item__
+ # (ignore mypy's expectation of __iter__ here)
+ data=[resource_item.convert_to_pandas_representation() for resource_item in self] # type: ignore
+ )
diff --git a/streampipes-client-python/streampipes_client/client/__init__.py b/streampipes-client-python/streampipes_client/model/resource/__init__.py
similarity index 89%
copy from streampipes-client-python/streampipes_client/client/__init__.py
copy to streampipes-client-python/streampipes_client/model/resource/__init__.py
index ecb1860df..ee6bb0da0 100644
--- a/streampipes-client-python/streampipes_client/client/__init__.py
+++ b/streampipes-client-python/streampipes_client/model/resource/__init__.py
@@ -13,4 +13,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
\ No newline at end of file
+#
+
+from .data_lake_measure import DataLakeMeasure
+
+__all__ = [
+ "DataLakeMeasure",
+]
diff --git a/streampipes-client-python/streampipes_client/model/resource/resource.py b/streampipes-client-python/streampipes_client/model/resource/resource.py
new file mode 100644
index 000000000..5cb2d354d
--- /dev/null
+++ b/streampipes-client-python/streampipes_client/model/resource/resource.py
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+General and abstract implementation for a resource.
+A resource defines the data model that is used by a resource container (`model.container.resourceContainer`).
+"""
+from abc import ABC, abstractmethod
+from typing import Dict
+
+from streampipes_client.model.common import BasicModel
+
+__all__ = [
+ "Resource",
+]
+
+
+class Resource(ABC, BasicModel):
+ """General and abstract implementation for a resource.
+ A resource defines the data model used by a resource container (`model.container.resourceContainer`).
+ It inherits from Pydantic's BaseModel to get all its superpowers,
+ which are used to parse, validate the API response and to easily switch between
+ the Python representation (both serialized and deserialized) and Java representation (serialized only).
+ """
+
+ @abstractmethod
+ def convert_to_pandas_representation(self) -> Dict:
+ """Returns a dictionary representation to be used when creating a pandas Dataframe."""
+ raise NotImplementedError # pragma: no cover
diff --git a/streampipes-client-python/tests/__init__.py b/streampipes-client-python/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/streampipes-client-python/streampipes_client/client/__init__.py b/streampipes-client-python/tests/client/__init__.py
similarity index 99%
copy from streampipes-client-python/streampipes_client/client/__init__.py
copy to streampipes-client-python/tests/client/__init__.py
index ecb1860df..cce3acad3 100644
--- a/streampipes-client-python/streampipes_client/client/__init__.py
+++ b/streampipes-client-python/tests/client/__init__.py
@@ -13,4 +13,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
\ No newline at end of file
+#
diff --git a/streampipes-client-python/tests/client/test_client.py b/streampipes-client-python/tests/client/test_client.py
new file mode 100644
index 000000000..8404da89e
--- /dev/null
+++ b/streampipes-client-python/tests/client/test_client.py
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from unittest import TestCase
+
+from streampipes_client.client import StreamPipesClient
+from streampipes_client.client.client_config import StreamPipesClientConfig
+from streampipes_client.client.credential_provider import StreamPipesApiKeyCredentials
+from streampipes_client.endpoint import DataLakeMeasureEndpoint
+
+
+class TestStreamPipesClient(TestCase):
+ def test_client_init(self):
+ result = StreamPipesClient(
+ client_config=StreamPipesClientConfig(
+ credential_provider=StreamPipesApiKeyCredentials(username="user", api_key="key"),
+ host_address="localhost",
+ )
+ )
+
+ expected_headers = {
+ "X-API-User": "user",
+ "X-API-Key": "key",
+ "Application": "application/json",
+ }
+ result_headers = dict(result.request_session.headers)
+ self.assertDictContainsSubset(
+ subset=expected_headers,
+ dictionary=result_headers,
+ )
+ self.assertTrue(isinstance(result.dataLakeMeasureApi, DataLakeMeasureEndpoint))
+ self.assertEqual(result.base_api_path, "https://localhost:80/streampipes-backend/")
+
+ def test_client_create(self):
+ result = StreamPipesClient.create(
+ client_config=StreamPipesClientConfig(
+ credential_provider=StreamPipesApiKeyCredentials(username="user", api_key="key"),
+ host_address="localhost",
+ https_disabled=True,
+ port=500,
+ )
+ )
+
+ expected_headers = {
+ "X-API-User": "user",
+ "X-API-Key": "key",
+ "Application": "application/json",
+ }
+ result_headers = dict(result.request_session.headers)
+ self.assertDictContainsSubset(
+ subset=expected_headers,
+ dictionary=result_headers,
+ )
+ self.assertTrue(isinstance(result.dataLakeMeasureApi, DataLakeMeasureEndpoint))
+ self.assertEqual(result.base_api_path, "http://localhost:500/streampipes-backend/")
diff --git a/streampipes-client-python/tests/client/test_credential_provider.py b/streampipes-client-python/tests/client/test_credential_provider.py
new file mode 100644
index 000000000..5490ded3f
--- /dev/null
+++ b/streampipes-client-python/tests/client/test_credential_provider.py
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+from unittest import TestCase
+
+from streampipes_client.client.credential_provider import StreamPipesApiKeyCredentials
+
+
+class TestStreamPipesApiKeyCredentials(TestCase):
+ def test_api_key_credentials(self):
+ credentials = StreamPipesApiKeyCredentials(username="test-user", api_key="test-key")
+ result = credentials.make_headers()
+
+ expected = {"X-API-User": "test-user", "X-API-Key": "test-key"}
+
+ self.assertDictEqual(expected, result)
+
+ result_extended = credentials.make_headers(http_headers={"test": "test"})
+ expected_extended = {**expected, "test": "test"}
+
+ self.assertDictEqual(expected_extended, result_extended)
+
+ def test_api_key_from_env(self):
+
+ os.environ["USER"] = "user"
+ os.environ["KEY"] = "api-key"
+ credentials = StreamPipesApiKeyCredentials.from_env(username_env="USER", api_key_env="KEY")
+
+ self.assertEqual("user", credentials.username)
+ self.assertEqual("api-key", credentials.api_key)
+
+ def test_api_key_from_env_not_set(self):
+
+ with self.assertRaises(KeyError):
+ StreamPipesApiKeyCredentials.from_env(username_env="test", api_key_env="key")