You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by bo...@apache.org on 2022/11/12 15:01:18 UTC
[incubator-streampipes] 03/18: tmp
This is an automated email from the ASF dual-hosted git repository.
bossenti pushed a commit to branch feature/STREAMPIPES-607
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 5d240655398bb250b5565aeddfc26d97d84cff59
Author: bossenti <bo...@posteo.de>
AuthorDate: Fri Nov 4 19:19:07 2022 +0100
tmp
---
streampipes-client-python/githooks/pre-commit | 10 ++
.../streampipesclient/client/__init__.py | 1 -
.../streampipesclient/client/client.py | 67 +++++++++++--
.../{client => endpoints}/__init__.py | 6 +-
.../endpoints/data_lake_measure.py | 18 ++++
.../streampipesclient/endpoints/endpoint.py | 104 +++++++++++++++++++++
.../{client => model}/__init__.py | 6 +-
.../streampipesclient/model/common.py | 85 +++++++++++++++++
.../{client => model/container}/__init__.py | 4 +-
.../model/container/data_lake_measures.py | 14 +++
.../model/container/model_container.py | 66 +++++++++++++
.../{client => model/element}/__init__.py | 4 +-
.../element/data_lake_measure.py} | 22 +++--
.../{client/config.py => model/element/element.py} | 17 +---
14 files changed, 378 insertions(+), 46 deletions(-)
diff --git a/streampipes-client-python/githooks/pre-commit b/streampipes-client-python/githooks/pre-commit
new file mode 100644
index 000000000..025fc6455
--- /dev/null
+++ b/streampipes-client-python/githooks/pre-commit
@@ -0,0 +1,10 @@
+#!/bin/sh
+set -e
+
+CHANGED_FILES="$(git --no-pager diff --name-status --no-color --cached --diff-filter=d|awk '$1 !~ /R/ && ( $2 ~ /\.py|\.pyi/ ) {print $2}; $1 ~ /R/ && ( $3 ~ /\.py|\.pyi/ ) {print $3}')"
+
+if [ -z "$CHANGED_FILES" ]; then
+ echo "No Python staged files."
+ exit 0
+fi;
+
diff --git a/streampipes-client-python/streampipesclient/client/__init__.py b/streampipes-client-python/streampipesclient/client/__init__.py
index 56ed17e8a..a323247a5 100644
--- a/streampipes-client-python/streampipesclient/client/__init__.py
+++ b/streampipes-client-python/streampipesclient/client/__init__.py
@@ -16,5 +16,4 @@
#
from .client import StreamPipesClient
-from .config import StreamPipesClientConfig
from .credentials import StreamPipesApiKeyCredentials
diff --git a/streampipes-client-python/streampipesclient/client/client.py b/streampipes-client-python/streampipesclient/client/client.py
index f2855bbeb..3df75869b 100644
--- a/streampipes-client-python/streampipesclient/client/client.py
+++ b/streampipes-client-python/streampipesclient/client/client.py
@@ -14,17 +14,72 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import logging
+import sys
+from dataclasses import dataclass
+from typing import Dict, Optional, Union, Type
+
+import requests
+
+from streampipesclient.client.credentials import CredentialProvider
+from streampipesclient.endpoints.data_lake_measure import DataLakeMeasureEndpoint
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class StreamPipesClientConfig:
+ """
+
+ """
+
+ credential_provider: CredentialProvider
+ host_address: str
+ https_disabled: Optional[bool] = False
+ port: Optional[int] = 80
+
class StreamPipesClient:
- def __init__(self):
+ def __init__(self,
+ *,
+ client_config: StreamPipesClientConfig,
+ logging_level: int = logging.INFO,
+ ):
+ self.client_config = client_config
+
+ self.request_session = requests.Session()
+ self.request_session.headers.update(self.http_headers)
+
+ self._set_up_logging(logging_level=logging_level)
+
+ # endpoints
+ self.dataLakeMeasureApi = DataLakeMeasureEndpoint(parent_client=self)
+
+ @staticmethod
+ def _set_up_logging(*, logging_level: int) -> None:
+ logging.basicConfig(
+ level=logging_level,
+ stream=sys.stdout,
+ format="%(asctime)s - %(name)s - [%(levelname)s] - [%(filename)s:%(lineno)d] [%(funcName)s] - %(message)s",
+ )
+
+ logger.info(f"Logging successfully initialized with logging level {logging.getLevelName(logging_level)}.")
- pass
+ @classmethod
+ def create(cls, *, client_config: StreamPipesClientConfig, logging_level: int = logging.INFO):
+ return cls(client_config=client_config, logging_level=logging_level)
@property
- def http_headers(self):
- pass
+ def http_headers(self) -> Dict[str, str]:
+ return self.client_config.credential_provider.make_headers(
+ {
+ "Application": "application/json"
+ }
+ )
@property
- def base_api_path(self):
- pass
\ No newline at end of file
+ def base_api_path(self) -> str:
+ return f"{'http://' if self.client_config.https_disabled else 'https://'}" \
+ f"{self.client_config.host_address}:" \
+ f"{self.client_config.port}/streampipes-backend/"
diff --git a/streampipes-client-python/streampipesclient/client/__init__.py b/streampipes-client-python/streampipesclient/endpoints/__init__.py
similarity index 84%
copy from streampipes-client-python/streampipesclient/client/__init__.py
copy to streampipes-client-python/streampipesclient/endpoints/__init__.py
index 56ed17e8a..ecb1860df 100644
--- a/streampipes-client-python/streampipesclient/client/__init__.py
+++ b/streampipes-client-python/streampipesclient/endpoints/__init__.py
@@ -13,8 +13,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
-
-from .client import StreamPipesClient
-from .config import StreamPipesClientConfig
-from .credentials import StreamPipesApiKeyCredentials
+#
\ No newline at end of file
diff --git a/streampipes-client-python/streampipesclient/endpoints/data_lake_measure.py b/streampipes-client-python/streampipesclient/endpoints/data_lake_measure.py
new file mode 100644
index 000000000..09c5e7fab
--- /dev/null
+++ b/streampipes-client-python/streampipesclient/endpoints/data_lake_measure.py
@@ -0,0 +1,18 @@
+from typing import List
+
+from streampipesclient.endpoints.endpoint import APIEndpoint
+from streampipesclient.model.container import ModelContainer
+from streampipesclient.model.container.data_lake_measures import DataLakeMeasures
+from streampipesclient.model.element import Element
+from streampipesclient.model.element.data_lake_measure import DataLakeMeasure
+
+
+class DataLakeMeasureEndpoint(APIEndpoint):
+
+ @classmethod
+ def _container_cls(cls):
+ return DataLakeMeasures
+
+ @property
+ def _relative_api_path(self):
+ return "api", "v4", "datalake", "measurements"
diff --git a/streampipes-client-python/streampipesclient/endpoints/endpoint.py b/streampipes-client-python/streampipesclient/endpoints/endpoint.py
new file mode 100644
index 000000000..b0a5c4693
--- /dev/null
+++ b/streampipes-client-python/streampipesclient/endpoints/endpoint.py
@@ -0,0 +1,104 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from http import HTTPStatus
+import logging
+from abc import ABC, abstractmethod
+from typing import Tuple, Type, Callable
+
+import requests
+from requests import Response
+from requests.exceptions import HTTPError
+
+from streampipesclient.model.container import ModelContainer
+from streampipesclient.model.element import Element
+
+logger = logging.getLogger(__name__)
+
+status_code_to_log_message = {
+ 401: "\nThe StreamPipes Backend returned an unauthorized error.\n"
+ "Please check your user name and/or password to be correct.",
+ 403: "\nThere seems to be an issue with the accees rights of the given user and the ressource you queried.\n"
+ "Apparently, this user is not allowd to query the resource.\n"
+ "Please check the user's permissions or contact your StreamPipes admin.",
+ **dict.fromkeys(
+ [404, 405],
+ "\nOops, there seems to be an issue with the Python Client calling the API inappropriately.\n"
+ "This should not happen, but unfortunately did.\n"
+ "If you don't mind, it would be awesome to let us know by creating an issue at github.com/apache/incubator-streampipes.\n"
+ "Please paste the following information to the issue description:\n\n")
+}
+
+
+class APIEndpoint(ABC):
+
+ def __init__(self, parent_client: "StreamPipesClient"):
+ self._parent_client = parent_client
+
+ @property
+ @abstractmethod
+ def _relative_api_path(self) -> Tuple[str]:
+ raise NotImplementedError
+
+ @classmethod
+ @abstractmethod
+ def _container_cls(cls) -> Type[ModelContainer]:
+ raise NotImplementedError
+
+ def _make_request(self,
+ *,
+ request_method: Callable[..., Response],
+ url: str,
+ ) -> Response:
+
+ response = request_method(url=url)
+
+ try:
+ response.raise_for_status()
+ except HTTPError as err:
+
+ status_code = err.response.status_code
+
+ log_message = status_code_to_log_message[err.response.status_code]
+
+ if status_code in [HTTPStatus.METHOD_NOT_ALLOWED.numerator, HTTPStatus.NOT_FOUND.numerator]:
+ log_message += f"url: {err.response.url}\nstatus code: {err.response.status_code}"
+
+ logger.debug(err.response.text)
+ raise HTTPError(log_message) from err
+
+ else:
+ logger.debug("Successfully retrieved resources from %s.", url)
+ logger.info("Successfully retrieved all resources.")
+
+ return response
+
+ def create_api_path(self) -> str:
+ return f"{self._parent_client.base_api_path}{'/'.join(api_path for api_path in self._relative_api_path)}"
+
+ def all(self) -> ModelContainer:
+
+ response = self._make_request(
+ request_method=self._parent_client.request_session.get,
+ url=self.create_api_path()
+ )
+ return self._container_cls().from_json(json_string=response.text)
+
+ def get(self, *, identifier: str) -> Element:
+
+ # equals to download
+ # needs further considerations
+ pass
diff --git a/streampipes-client-python/streampipesclient/client/__init__.py b/streampipes-client-python/streampipesclient/model/__init__.py
similarity index 84%
copy from streampipes-client-python/streampipesclient/client/__init__.py
copy to streampipes-client-python/streampipesclient/model/__init__.py
index 56ed17e8a..ecb1860df 100644
--- a/streampipes-client-python/streampipesclient/client/__init__.py
+++ b/streampipes-client-python/streampipesclient/model/__init__.py
@@ -13,8 +13,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
-
-from .client import StreamPipesClient
-from .config import StreamPipesClientConfig
-from .credentials import StreamPipesApiKeyCredentials
+#
\ No newline at end of file
diff --git a/streampipes-client-python/streampipesclient/model/common.py b/streampipes-client-python/streampipesclient/model/common.py
new file mode 100644
index 000000000..b1994372f
--- /dev/null
+++ b/streampipes-client-python/streampipesclient/model/common.py
@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import List, Optional
+
+from pydantic import BaseModel, StrictStr, StrictInt, StrictBool, ValidationError
+
+
+def snake_to_camel_case(snake_case_string: str) -> str:
+ """
+ Converts a string in snake_case format to camelCase style.
+ """
+
+ tokens = snake_case_string.split("_")
+
+ return tokens[0] + "".join(t.title() for t in tokens[1:])
+
+
+class BasicModel(BaseModel):
+ element_id: Optional[StrictStr]
+
+ class Config:
+ alias_generator = snake_to_camel_case
+
+
+class EventPropertyQualityDefinition(BasicModel):
+ pass
+
+
+class EventPropertyQualityRequirement(BasicModel):
+ minimum_property_quality: Optional[EventPropertyQualityDefinition]
+ maximum_property_quality: Optional[EventPropertyQualityDefinition]
+
+
+class EventProperty(BasicModel):
+ label: StrictStr
+ description: StrictStr
+ runtime_name: StrictStr
+ required: StrictBool
+ domain_properties: List[StrictStr]
+ event_property_qualities: List[EventPropertyQualityDefinition]
+ requires_event_property_qualities: List[EventPropertyQualityRequirement]
+ property_scope: Optional[StrictStr]
+ index: StrictInt
+ runtime_id: Optional[StrictStr]
+ runtime_type: Optional[StrictStr]
+ measurement_unit: Optional[StrictStr]
+ value_specification: Optional[StrictStr]
+
+
+class EventSchema(BasicModel):
+ event_properties: List[EventProperty]
+
+
+class StreamPipesDataModelError(RuntimeError):
+
+ def __init__(self, validation_error: ValidationError):
+ super().__init__(
+ self._generate_error_message(model=validation_error.model,
+ error_description=validation_error.json(),
+ )
+ )
+
+ @staticmethod
+ def _generate_error_message(*, model: BasicModel, error_description: str) -> str:
+ return f"\nOops, there seems to be a problem with our internal StreamPipes data model.\n" \
+ f"This should not occur, but unfortunately did.\n" \
+ f"Therefore, it would be great if you could report this problem as an issue at github.com/apache/incubator-streampipes.\n" \
+ f"Please don't forget to include the following information:\n\n" \
+ f"Affected Model class: {str(model)}\n" \
+ f"Validation error log: {error_description}"
diff --git a/streampipes-client-python/streampipesclient/client/__init__.py b/streampipes-client-python/streampipesclient/model/container/__init__.py
similarity index 85%
copy from streampipes-client-python/streampipesclient/client/__init__.py
copy to streampipes-client-python/streampipesclient/model/container/__init__.py
index 56ed17e8a..9d8d2b197 100644
--- a/streampipes-client-python/streampipesclient/client/__init__.py
+++ b/streampipes-client-python/streampipesclient/model/container/__init__.py
@@ -15,6 +15,4 @@
# limitations under the License.
#
-from .client import StreamPipesClient
-from .config import StreamPipesClientConfig
-from .credentials import StreamPipesApiKeyCredentials
+from .model_container import ModelContainer
diff --git a/streampipes-client-python/streampipesclient/model/container/data_lake_measures.py b/streampipes-client-python/streampipesclient/model/container/data_lake_measures.py
new file mode 100644
index 000000000..11005510a
--- /dev/null
+++ b/streampipes-client-python/streampipesclient/model/container/data_lake_measures.py
@@ -0,0 +1,14 @@
+import pandas as pd
+
+from streampipesclient.model.container import ModelContainer
+from streampipesclient.model.element.data_lake_measure import DataLakeMeasure
+
+
+class DataLakeMeasures(ModelContainer):
+
+ @classmethod
+ def _element_cls(cls):
+ return DataLakeMeasure
+
+ def to_pandas(self) -> pd.DataFrame:
+ pass
\ No newline at end of file
diff --git a/streampipes-client-python/streampipesclient/model/container/model_container.py b/streampipes-client-python/streampipesclient/model/container/model_container.py
new file mode 100644
index 000000000..666f06ac9
--- /dev/null
+++ b/streampipes-client-python/streampipesclient/model/container/model_container.py
@@ -0,0 +1,66 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import json
+
+import pandas as pd
+
+from abc import ABC, abstractmethod
+from typing import List, Type
+
+from pydantic import ValidationError
+
+from streampipesclient.model.common import StreamPipesDataModelError
+from streampipesclient.model.element import Element
+
+
+class ModelContainer(ABC):
+
+ def __init__(self, elements: List[Element]):
+ self._elements = elements
+
+ def __getitem__(self, position: int) -> Element:
+ return self._elements[position]
+
+ def __len__(self) -> int:
+ return len(self._elements)
+
+ @classmethod
+ def from_json(cls, json_string: str) -> "ModelContainer":
+
+ data = json.loads(json_string)
+
+ if not type(data) == list:
+ raise RuntimeError
+
+ try:
+
+ model_container = cls(
+ elements=[cls._element_cls().parse_obj(list_item) for list_item in data]
+ )
+ except ValidationError as ve:
+ raise StreamPipesDataModelError(validation_error=ve)
+
+ return model_container
+
+ @abstractmethod
+ def to_pandas(self) -> pd.DataFrame:
+ raise NotImplementedError
+
+ @classmethod
+ @abstractmethod
+ def _element_cls(cls) -> Type[Element]:
+ raise NotImplementedError
diff --git a/streampipes-client-python/streampipesclient/client/__init__.py b/streampipes-client-python/streampipesclient/model/element/__init__.py
similarity index 85%
copy from streampipes-client-python/streampipesclient/client/__init__.py
copy to streampipes-client-python/streampipesclient/model/element/__init__.py
index 56ed17e8a..f4b8a3ece 100644
--- a/streampipes-client-python/streampipesclient/client/__init__.py
+++ b/streampipes-client-python/streampipesclient/model/element/__init__.py
@@ -15,6 +15,4 @@
# limitations under the License.
#
-from .client import StreamPipesClient
-from .config import StreamPipesClientConfig
-from .credentials import StreamPipesApiKeyCredentials
+from .element import Element
\ No newline at end of file
diff --git a/streampipes-client-python/streampipesclient/client/client.py b/streampipes-client-python/streampipesclient/model/element/data_lake_measure.py
similarity index 64%
copy from streampipes-client-python/streampipesclient/client/client.py
copy to streampipes-client-python/streampipesclient/model/element/data_lake_measure.py
index f2855bbeb..7601bc115 100644
--- a/streampipes-client-python/streampipesclient/client/client.py
+++ b/streampipes-client-python/streampipesclient/model/element/data_lake_measure.py
@@ -14,17 +14,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+from typing import Optional
-class StreamPipesClient:
+from pydantic import StrictStr, StrictBool
- def __init__(self):
+from streampipesclient.model.common import EventSchema
+from streampipesclient.model.element import Element
- pass
- @property
- def http_headers(self):
- pass
-
- @property
- def base_api_path(self):
- pass
\ No newline at end of file
+class DataLakeMeasure(Element):
+ measure_name: StrictStr
+ timestamp_field: StrictStr
+ event_schema: EventSchema
+ pipeline_id: Optional[StrictStr]
+ pipeline_name: Optional[StrictStr]
+ pipeline_is_running: StrictBool
+ schema_version: StrictStr
diff --git a/streampipes-client-python/streampipesclient/client/config.py b/streampipes-client-python/streampipesclient/model/element/element.py
similarity index 69%
rename from streampipes-client-python/streampipesclient/client/config.py
rename to streampipes-client-python/streampipesclient/model/element/element.py
index b1f62fe24..bff745778 100644
--- a/streampipes-client-python/streampipesclient/client/config.py
+++ b/streampipes-client-python/streampipesclient/model/element/element.py
@@ -14,19 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from dataclasses import dataclass
-from typing import Optional, Type
+from abc import ABC
-from streampipesclient.client.credentials import CredentialProvider
+from streampipesclient.model.common import BasicModel
-@dataclass
-class StreamPipesClientConfig:
- """
-
- """
-
- credential_provider: Type[CredentialProvider]
- host_address: str
- https_disabled: Optional[bool] = False
- port: Optional[int] = 80
+class Element(ABC, BasicModel):
+ pass