You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/04/06 15:24:36 UTC
[streampipes] branch SP-1367 updated (4c282c8d1 -> 1749eb44d)
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a change to branch SP-1367
in repository https://gitbox.apache.org/repos/asf/streampipes.git
from 4c282c8d1 Merge branch 'dev' into SP-1367
add 5000139e0 Bump error_prone_annotations from 2.2.0 to 2.18.0 (#1403)
add 3ca5f9b21 Bump @typescript-eslint/parser from 5.54.0 to 5.56.0 in /ui
add 792f2d108 Bump eslint from 8.36.0 to 8.37.0 in /ui
add c8fd621db Bump kafka.version from 3.3.1 to 3.4.0
add 45b27818e Bump black from 23.1.0 to 23.3.0 in /streampipes-client-python
add 004422938 Bump boofcv-core from 0.33 to 0.42
add 4f74f9428 [hotfix] Remove duplicate declaration of snakeyaml dependency version
add c307a7bd2 [hotfix] Temporarily revert to snakeyaml 1.33
add c33607da7 [hotfix] Fix configuration for pipeline element development (#1465)
add 89d183d9c Merge branch 'dev' of github.com:apache/streampipes into dev
add 8503549cf Implement create method in Java PipelineApi (#1467)
add dac406332 Bump activemq-client from 5.17.3 to 5.18.0
add 51db2ee3f Bump spring-core from 6.0.3 to 6.0.7
add a201c5c05 Bump spring boot version
add a771e9f1b Bump Guava version, Bump spring-security version
add adc1d62af Add properties-migrator to dependency management
add c18643663 Improve data explorer query management (#1406) (#1407)
add eb4b9cbc2 Migrate KafkaPublisher (#1473)
add c627add09 Bump jasmine-core from 4.5.0 to 4.6.0 in /ui (#1474)
add 49c2f53b2 Bump @typescript-eslint/eslint-plugin from 5.55.0 to 5.57.1 in /ui
add fa69191b5 Provide database name in query object (#1479)
add cdcd83ea2 [#1289] Fix cypress test for data lake configuration (#1480)
add 1dcc11b5b [hotfix] Improve handling of count queries, improve table widget (#1484)
add 07494bf59 Bump mkdocstrings[python] in /streampipes-client-python (#1487)
new 3603a27e3 Merge branch 'dev' into SP-1367
new 1749eb44d [SP-1367] Use `QueryResult` as return type for datalakemeasure API
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
installer/cli/environments/pipeline-element | 2 -
installer/cli/environments/pipeline-element-nats | 2 -
pom.xml | 28 +-
...cting-data-from-the-streampipes-data-lake.ipynb | 4 +-
streampipes-client-python/setup.py | 4 +-
.../streampipes/endpoint/api/data_lake_measure.py | 1 -
.../streampipes/model/resource/data_lake_series.py | 13 +-
.../streampipes/model/resource/exceptions.py | 16 +
.../streampipes/model/resource/query_result.py | 53 ++--
streampipes-client-python/streampipes/test.py | 17 --
.../tests/client/test_data_lake_series.py | 48 +--
.../apache/streampipes/client/api/PipelineApi.java | 5 +-
.../commons/configs/CouchDbConfigurations.java | 35 ---
.../commons/configs/CouchDbEnvKeys.java | 25 --
.../configs/DataExplorerConfigurations.java | 46 ---
.../commons/configs/DataExplorerEnvKeys.java | 28 --
.../dataexplorer/DataExplorerQueryManagement.java | 142 +++++++++
...ntV3.java => DataExplorerSchemaManagement.java} | 93 ++++--
.../dataexplorer/DataLakeManagementV4.java | 245 ---------------
.../api/IDataExplorerQueryManagement.java | 47 +++
.../api/IDataExplorerSchemaManagement.java} | 24 +-
.../IQueryStatement.java} | 14 +-
.../influx/DataExplorerInfluxQueryExecutor.java | 169 +++++++++++
.../DataLakeInfluxQueryBuilder.java} | 138 +++++----
.../DeleteQueryParams.java} | 45 ++-
.../param/ProvidedRestQueryParamConverter.java | 147 +++++++++
.../ProvidedRestQueryParams.java} | 12 +-
.../dataexplorer/param/SelectQueryParams.java | 140 +++++++++
.../SupportedRestQueryParams.java} | 4 +-
.../model/AggregationFunction.java} | 8 +-
.../model/FillClauseParams.java} | 20 +-
.../model/GroupByTagsClauseParams.java} | 24 +-
.../model/GroupByTimeClauseParams.java} | 19 +-
.../model/LimitClauseParams.java} | 19 +-
.../model/OffsetClauseParams.java} | 19 +-
.../model/OrderByClauseParams.java} | 20 +-
.../model/SelectClauseParams.java} | 73 ++---
.../{v4/params => param/model}/SelectColumn.java | 58 ++--
.../param/model/WhereClauseParams.java | 122 ++++++++
.../{v4 => query}/AutoAggregationHandler.java | 53 ++--
.../query/DataExplorerQueryExecutor.java | 106 +++++++
.../{v4 => }/query/QueryResultProvider.java | 28 +-
.../query/StreamedQueryResultProvider.java | 18 +-
.../query/writer/ConfiguredCsvOutputWriter.java | 10 +-
.../query/writer/ConfiguredJsonOutputWriter.java | 10 +-
.../query/writer/ConfiguredOutputWriter.java | 8 +-
.../{v4 => }/query/writer/OutputFormat.java | 2 +-
.../{v4 => }/query/writer/item/CsvItemWriter.java | 2 +-
.../{v4 => }/query/writer/item/ItemGenerator.java | 4 +-
.../{v4 => }/query/writer/item/JsonItemWriter.java | 2 +-
.../DataLakeQueryOrdering.java | 2 +-
.../FilterCondition.java} | 24 +-
.../querybuilder/IDataLakeQueryBuilder.java | 84 ++++++
.../dataexplorer/sdk/DataLakeQueryConstants.java | 30 --
.../dataexplorer/sdk/IDataLakeQueryBuilder.java | 74 -----
.../dataexplorer/{v4 => }/utils/TimeParser.java | 2 +-
.../v4/params/DeleteFromStatementParams.java | 31 --
.../dataexplorer/v4/params/FillParams.java | 35 ---
.../v4/params/WhereStatementParams.java | 122 --------
.../dataexplorer/v4/query/DataExplorerQueryV4.java | 234 --------------
.../dataexplorer/v4/query/QueryBuilder.java | 64 ----
.../v4/query/elements/DeleteFromStatement.java | 33 --
.../v4/query/elements/GroupingByTags.java | 43 ---
.../v4/query/elements/GroupingByTime.java | 34 ---
.../v4/query/elements/ItemLimitation.java | 34 ---
.../dataexplorer/v4/query/elements/Offset.java | 34 ---
.../v4/query/elements/OrderingByTime.java | 34 ---
.../v4/query/elements/QueryElement.java | 35 ---
.../v4/query/elements/SelectFromStatement.java | 51 ----
.../v4/query/elements/TimeBoundary.java | 40 ---
.../v4/query/elements/WhereStatement.java | 39 ---
.../dataexplorer/v4/template/QueryTemplatesV4.java | 63 ----
.../v4/utils/DataLakeManagementUtils.java | 163 ----------
.../dataexplorer/param/SelectQueryParamsTest.java | 208 +++++++++++++
.../param}/WhereStatementParamsTest.java | 27 +-
.../writer/TestConfiguredCsvOutputWriter.java | 7 +-
.../writer/TestConfiguredJsonOutputWriter.java | 7 +-
.../query/writer/TestConfiguredOutputWriter.java | 2 +-
.../query/writer/item/TestCsvItemWriter.java | 4 +-
.../query/writer/item/TestItemWriter.java | 2 +-
.../query/writer/item/TestJsonItemWriter.java | 4 +-
.../dataexplorer/sdk/DataLakeQueryBuilderTest.java | 6 +-
.../utils/ProvidedQueryParameterBuilder.java | 92 ++++++
.../streampipes-connect-adapters-iiot/pom.xml | 4 -
.../streampipes-connect-adapters/pom.xml | 4 -
.../sinks/brokers/jvm/BrokersJvmInit.java | 4 +-
.../sinks/brokers/jvm/kafka/KafkaController.java | 91 ------
.../sinks/brokers/jvm/kafka/KafkaParameters.java | 56 ++--
.../sinks/brokers/jvm/kafka/KafkaPublishSink.java | 130 ++++++++
.../sinks/brokers/jvm/kafka/KafkaPublisher.java | 86 ------
.../sinks/internal/jvm/SinksInternalJvmInit.java | 4 -
.../streampipes/ps/DataLakeMeasureResourceV3.java | 57 ----
.../streampipes/ps/DataLakeMeasureResourceV4.java | 23 +-
.../apache/streampipes/ps/DataLakeResourceV4.java | 70 +++--
.../apache/streampipes/rest/ResetManagement.java | 14 +-
streampipes-service-core/pom.xml | 1 -
.../service/core/StreamPipesResourceConfig.java | 2 -
ui/cypress/support/utils/PrepareTestDataUtils.ts | 8 +-
.../tests/datalake/configuration.smoke.spec.ts | 7 +-
ui/package-lock.json | 335 +++++++++++----------
ui/package.json | 8 +-
.../widgets/table/table-widget.component.html | 1 +
102 files changed, 2242 insertions(+), 2499 deletions(-)
create mode 100644 streampipes-client-python/streampipes/model/resource/exceptions.py
delete mode 100644 streampipes-client-python/streampipes/test.py
delete mode 100644 streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/configs/CouchDbConfigurations.java
delete mode 100644 streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/configs/CouchDbEnvKeys.java
delete mode 100644 streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/configs/DataExplorerConfigurations.java
delete mode 100644 streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/configs/DataExplorerEnvKeys.java
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerQueryManagement.java
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{DataLakeNoUserManagementV3.java => DataExplorerSchemaManagement.java} (54%)
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerQueryManagement.java
copy streampipes-data-explorer/src/{test/java/org/apache/streampipesdataexplorer/v4/query/writer/item/TestItemWriter.java => main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java} (64%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/params/QueryParamsV4.java => api/IQueryStatement.java} (76%)
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{sdk/DataLakeQueryBuilder.java => influx/DataLakeInfluxQueryBuilder.java} (55%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/params/TimeBoundaryParams.java => param/DeleteQueryParams.java} (50%)
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParamConverter.java
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/ProvidedQueryParams.java => param/ProvidedRestQueryParams.java} (86%)
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SelectQueryParams.java
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/SupportedDataLakeQueryParameters.java => param/SupportedRestQueryParams.java} (96%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/params/ColumnFunction.java => param/model/AggregationFunction.java} (86%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/query/elements/FillStatement.java => param/model/FillClauseParams.java} (62%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/params/GroupingByTagsParams.java => param/model/GroupByTagsClauseParams.java} (56%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/params/GroupingByTimeParams.java => param/model/GroupByTimeClauseParams.java} (61%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/params/ItemLimitationParams.java => param/model/LimitClauseParams.java} (64%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/params/OffsetParams.java => param/model/OffsetClauseParams.java} (64%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/params/OrderingByTimeParams.java => param/model/OrderByClauseParams.java} (58%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/params/SelectFromStatementParams.java => param/model/SelectClauseParams.java} (50%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/params => param/model}/SelectColumn.java (53%)
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4 => query}/AutoAggregationHandler.java (68%)
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataExplorerQueryExecutor.java
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4 => }/query/QueryResultProvider.java (61%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4 => }/query/StreamedQueryResultProvider.java (84%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4 => }/query/writer/ConfiguredCsvOutputWriter.java (86%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4 => }/query/writer/ConfiguredJsonOutputWriter.java (85%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4 => }/query/writer/ConfiguredOutputWriter.java (89%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4 => }/query/writer/OutputFormat.java (95%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4 => }/query/writer/item/CsvItemWriter.java (94%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4 => }/query/writer/item/ItemGenerator.java (92%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4 => }/query/writer/item/JsonItemWriter.java (95%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{sdk => querybuilder}/DataLakeQueryOrdering.java (93%)
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4/params/WhereCondition.java => querybuilder/FilterCondition.java} (70%)
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/IDataLakeQueryBuilder.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/sdk/DataLakeQueryConstants.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/sdk/IDataLakeQueryBuilder.java
rename streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/{v4 => }/utils/TimeParser.java (97%)
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/DeleteFromStatementParams.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/FillParams.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/WhereStatementParams.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/DataExplorerQueryV4.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/QueryBuilder.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/DeleteFromStatement.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/GroupingByTags.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/GroupingByTime.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/ItemLimitation.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/Offset.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/OrderingByTime.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/QueryElement.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/SelectFromStatement.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/TimeBoundary.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/WhereStatement.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/template/QueryTemplatesV4.java
delete mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/utils/DataLakeManagementUtils.java
create mode 100644 streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/SelectQueryParamsTest.java
rename {streampipes-rest/src/test/java/org/apache/streampipes/dataexplorer/v4/params => streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param}/WhereStatementParamsTest.java (53%)
rename streampipes-data-explorer/src/test/java/org/apache/{streampipesdataexplorer/v4 => streampipes/dataexplorer}/query/writer/TestConfiguredCsvOutputWriter.java (85%)
rename streampipes-data-explorer/src/test/java/org/apache/{streampipesdataexplorer/v4 => streampipes/dataexplorer}/query/writer/TestConfiguredJsonOutputWriter.java (85%)
rename streampipes-data-explorer/src/test/java/org/apache/{streampipesdataexplorer/v4 => streampipes/dataexplorer}/query/writer/TestConfiguredOutputWriter.java (95%)
rename streampipes-data-explorer/src/test/java/org/apache/{streampipesdataexplorer/v4 => streampipes/dataexplorer}/query/writer/item/TestCsvItemWriter.java (91%)
rename streampipes-data-explorer/src/test/java/org/apache/{streampipesdataexplorer/v4 => streampipes/dataexplorer}/query/writer/item/TestItemWriter.java (94%)
rename streampipes-data-explorer/src/test/java/org/apache/{streampipesdataexplorer/v4 => streampipes/dataexplorer}/query/writer/item/TestJsonItemWriter.java (89%)
create mode 100644 streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/utils/ProvidedQueryParameterBuilder.java
delete mode 100644 streampipes-extensions/streampipes-sinks-brokers-jvm/src/main/java/org/apache/streampipes/sinks/brokers/jvm/kafka/KafkaController.java
create mode 100644 streampipes-extensions/streampipes-sinks-brokers-jvm/src/main/java/org/apache/streampipes/sinks/brokers/jvm/kafka/KafkaPublishSink.java
delete mode 100644 streampipes-extensions/streampipes-sinks-brokers-jvm/src/main/java/org/apache/streampipes/sinks/brokers/jvm/kafka/KafkaPublisher.java
delete mode 100644 streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeMeasureResourceV3.java
[streampipes] 02/02: [SP-1367] Use `QueryResult` as return type for datalakemeasure API
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch SP-1367
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 1749eb44d55dc7b2ab3b138838799cf1a8829b94
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Thu Apr 6 17:24:24 2023 +0200
[SP-1367] Use `QueryResult` as return type for datalakemeasure API
---
...cting-data-from-the-streampipes-data-lake.ipynb | 4 +-
.../streampipes/endpoint/api/data_lake_measure.py | 1 -
.../streampipes/model/resource/data_lake_series.py | 13 +-----
.../streampipes/model/resource/exceptions.py | 16 +++++++
.../streampipes/model/resource/query_result.py | 53 ++++++++++++++--------
streampipes-client-python/streampipes/test.py | 17 -------
.../tests/client/test_data_lake_series.py | 48 ++++++++++++--------
7 files changed, 83 insertions(+), 69 deletions(-)
diff --git a/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb b/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb
index 6f1d7d81d..66a71f08b 100644
--- a/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb
+++ b/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb
@@ -370,8 +370,8 @@
},
{
"data": {
- "text/plain": " time mass_flow temperature\n0 2023-02-24T16:19:41.472Z 3.309556 44.448483\n1 2023-02-24T16:19:41.482Z 5.608580 40.322033\n2 2023-02-24T16:19:41.493Z 7.692881 49.239639\n3 2023-02-24T16:19:41.503Z 3.632898 49.933754\n4 2023-02-24T16:19:41.513Z 0.711260 50.106617\n.. ... ... ...\n995 2023-02-24T16:19:52.927Z 1.740114 46.558231\n996 2023-02-24T16:19:52.94Z [...]
- "text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>time</th>\n <th>mass_flow</th>\n <th>temperature</th>\n </tr>\n </thead>\n <tbody>\n < [...]
+ "text/plain": " timestamp mass_flow temperature\n0 2023-02-24T16:19:41.472Z 3.309556 44.448483\n1 2023-02-24T16:19:41.482Z 5.608580 40.322033\n2 2023-02-24T16:19:41.493Z 7.692881 49.239639\n3 2023-02-24T16:19:41.503Z 3.632898 49.933754\n4 2023-02-24T16:19:41.513Z 0.711260 50.106617\n.. ... ... ...\n995 2023-02-24T16:19:52.927Z 1.740114 46.558231\n996 2023-02-24T16:19:52. [...]
+ "text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>timestamp</th>\n <th>mass_flow</th>\n <th>temperature</th>\n </tr>\n </thead>\n <tbody>\n [...]
},
"execution_count": 14,
"metadata": {},
diff --git a/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py b/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py
index 0fbbfced8..c7493683c 100644
--- a/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py
+++ b/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py
@@ -26,7 +26,6 @@ from pydantic import BaseModel, Extra, Field, StrictInt, ValidationError, valida
from streampipes.endpoint.endpoint import APIEndpoint
from streampipes.model.container import DataLakeMeasures
from streampipes.model.container.resource_container import ResourceContainer
-from streampipes.model.resource import DataSeries
__all__ = [
"DataLakeMeasureEndpoint",
diff --git a/streampipes-client-python/streampipes/model/resource/data_lake_series.py b/streampipes-client-python/streampipes/model/resource/data_lake_series.py
index b68e1763a..7bb4ffaaa 100644
--- a/streampipes-client-python/streampipes/model/resource/data_lake_series.py
+++ b/streampipes-client-python/streampipes/model/resource/data_lake_series.py
@@ -22,6 +22,7 @@ from typing import Any, Dict, List, Optional, Union
import pandas as pd
from pydantic import StrictInt, StrictStr
+from streampipes.model.resource.exceptions import StreamPipesUnsupportedDataLakeSeries
from streampipes.model.resource.resource import Resource
__all__ = [
@@ -29,18 +30,6 @@ __all__ = [
]
-class StreamPipesUnsupportedDataLakeSeries(Exception):
- """Exception to be raised when the returned data lake series
- cannot be parsed with the current implementation of the resource.
- """
-
- def __init__(self):
- super().__init__(
- "The Data Lake series returned by the API appears "
- "to have a structure that is not currently supported by the Python client."
- )
-
-
class DataSeries(Resource):
"""Implementation of a resource for data lake series.
This resource defines the data model used by its resource container(`model.container.DataLakeMeasures`).
diff --git a/streampipes-client-python/streampipes/model/resource/exceptions.py b/streampipes-client-python/streampipes/model/resource/exceptions.py
new file mode 100644
index 000000000..5181a3bea
--- /dev/null
+++ b/streampipes-client-python/streampipes/model/resource/exceptions.py
@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from typing import Optional
+
+
+class StreamPipesUnsupportedDataLakeSeries(Exception):
+ """Exception to be raised when the returned data lake series
+ cannot be parsed with the current implementation of the resource.
+ """
+
+ def __init__(self, reason: Optional[str] = None):
+ super().__init__(
+ "The Data Lake series returned by the API appears "
+ "to have a structure that is not currently supported by the Python client."
+ f"Reason: {reason}"
+ )
diff --git a/streampipes-client-python/streampipes/model/resource/query_result.py b/streampipes-client-python/streampipes/model/resource/query_result.py
index d7b91cc50..1020902aa 100644
--- a/streampipes-client-python/streampipes/model/resource/query_result.py
+++ b/streampipes-client-python/streampipes/model/resource/query_result.py
@@ -17,14 +17,13 @@
from __future__ import annotations
-import json
from itertools import chain
-from typing import Any, Dict, List, Optional, Union, Literal
+from typing import Any, Dict, List, Literal, Union
import pandas as pd
-from pydantic import StrictInt, StrictStr, Field
-
+from pydantic import Field, StrictInt, StrictStr
from streampipes.model.resource import DataSeries
+from streampipes.model.resource.exceptions import StreamPipesUnsupportedDataLakeSeries
from streampipes.model.resource.resource import Resource
__all__ = [
@@ -33,6 +32,12 @@ __all__ = [
class QueryResult(Resource):
+ """Implementation of a resource for query result.
+ This resource defines the data model used by its resource container(`model.container.DataLakeMeasures`).
+ It inherits from Pydantic's BaseModel to get all its superpowers,
+ which are used to parse, validate the API response and to easily switch between
+ the Python representation (both serialized and deserialized) and Java representation (serialized only).
+ """
def convert_to_pandas_representation(self) -> Dict[str, Union[List[str], List[List[Any]]]]:
"""Returns the dictionary representation of a data lake series
@@ -42,35 +47,45 @@ class QueryResult(Resource):
Returns
-------
- Dictionary
+ dict
Dictionary with the keys `headers` and `rows`
+ Raises
+ ------
+ StreamPipesUnsupportedDataLakeSeries
+ If the query result returned by the StreamPipes API cannot be converted to the pandas representation
+
"""
- return self.dict(include={"headers", "rows"})
+ for series in self.all_data_series:
+ if self.headers != series.headers:
+ raise StreamPipesUnsupportedDataLakeSeries("Headers of series does not match query result headers")
+
+ if self.headers[0] == "time":
+ self.headers[0] = "timestamp"
+ else:
+ raise StreamPipesUnsupportedDataLakeSeries(f"Unsupported headers {self.headers}")
+
+ return {
+ "headers": self.headers,
+ "rows": list(chain.from_iterable([series.rows for series in self.all_data_series])),
+ }
total: StrictInt
headers: List[StrictStr]
all_data_series: List[DataSeries]
- query_status: Literal['OK', 'TOO_MUCH_DATA'] = Field(alias="spQueryStatus")
+ query_status: Literal["OK", "TOO_MUCH_DATA"] = Field(alias="spQueryStatus")
def to_pandas(self) -> pd.DataFrame:
"""Returns the data lake series in representation of a Pandas Dataframe.
Returns
-------
- pd.DataFrame
+ df: pd.DataFrame
+ Pandas df containing the query result
"""
- # Pseudocode:
- # pandas_representation = self.convert_to_pandas_representation()
- #
- # pd = pd.DataFrame(data=chain(*[item.rows for item in self.all_data_series]),
- # columns=pandas_representation["headers"])
- #
- # if tags not None:
- # pd.groupBy(tags)
-
- return pd
+ pandas_representation = self.convert_to_pandas_representation()
+ df = pd.DataFrame(data=pandas_representation["rows"], columns=pandas_representation["headers"])
- test ={})
\ No newline at end of file
+ return df
diff --git a/streampipes-client-python/streampipes/test.py b/streampipes-client-python/streampipes/test.py
deleted file mode 100644
index 97bceaa72..000000000
--- a/streampipes-client-python/streampipes/test.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from streampipes.client import StreamPipesClient
-from streampipes.client.config import StreamPipesClientConfig
-from streampipes.client.credential_provider import StreamPipesApiKeyCredentials
-
-if __name__ == '__main__':
- config = StreamPipesClientConfig(
- credential_provider=StreamPipesApiKeyCredentials(
- username="admin@streampipes.apache.org",
- api_key="Mbias0Uqdytro5fMEMnXXBYM",
- ),
- host_address="localhost",
- https_disabled=True,
- port=8082
- )
-
- client = StreamPipesClient(client_config=config)
- print(client.dataLakeMeasureApi.get('test').to_pandas())
diff --git a/streampipes-client-python/tests/client/test_data_lake_series.py b/streampipes-client-python/tests/client/test_data_lake_series.py
index 694104773..21a3afc53 100644
--- a/streampipes-client-python/tests/client/test_data_lake_series.py
+++ b/streampipes-client-python/tests/client/test_data_lake_series.py
@@ -21,25 +21,25 @@ from unittest.mock import MagicMock, call, patch
from streampipes.client import StreamPipesClient
from streampipes.client.config import StreamPipesClientConfig
from streampipes.client.credential_provider import StreamPipesApiKeyCredentials
-from streampipes.model.resource.data_lake_series import (
- StreamPipesUnsupportedDataLakeSeries,
-)
+from streampipes.model.resource.exceptions import StreamPipesUnsupportedDataLakeSeries
class TestDataLakeSeries(TestCase):
def setUp(self) -> None:
+ self.base_headers = [
+ "changeDetectedHigh",
+ "changeDetectedLow",
+ "cumSumHigh",
+ "cumSumLow",
+ "level",
+ "overflow",
+ "sensorId",
+ "underflow",
+ ]
- self.headers = [
- "time",
- "changeDetectedHigh",
- "changeDetectedLow",
- "cumSumHigh",
- "cumSumLow",
- "level",
- "overflow",
- "sensorId",
- "underflow",
- ]
+ self.headers = ["time"] + self.base_headers
+
+ self.headers_expected = ["timestamp"] + self.base_headers
self.data_series = {
"total": 2,
@@ -93,10 +93,8 @@ class TestDataLakeSeries(TestCase):
return result.to_pandas()
-
@patch("streampipes.client.client.Session", autospec=True)
def test_to_pandas(self, http_session: MagicMock):
-
query_result = {
"total": 1,
"headers": self.headers,
@@ -110,7 +108,7 @@ class TestDataLakeSeries(TestCase):
self.assertEqual(2, len(result_pd))
self.assertListEqual(
- self.headers,
+ self.headers_expected,
list(result_pd.columns),
)
self.assertEqual(73.37740325927734, result_pd["level"][0])
@@ -131,7 +129,21 @@ class TestDataLakeSeries(TestCase):
self.assertEqual(4, len(result_pd))
self.assertListEqual(
- self.headers,
+ self.headers_expected,
list(result_pd.columns),
)
self.assertEqual(70.03279876708984, result_pd["level"][3])
+
+ @patch("streampipes.client.client.Session", autospec=True)
+ def test_different_headers_exception(self, http_session: MagicMock):
+ query_result = {
+ "total": 1,
+ "headers": ['one'],
+ "spQueryStatus": "OK",
+ "allDataSeries": [
+ self.data_series
+ ],
+ }
+
+ with self.assertRaises(StreamPipesUnsupportedDataLakeSeries):
+ self.get_result_as_panda(http_session, query_result)
[streampipes] 01/02: Merge branch 'dev' into SP-1367
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch SP-1367
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 3603a27e3674f9414495c7e6682c69e4c47f708a
Merge: 4c282c8d1 07494bf59
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Thu Apr 6 16:06:02 2023 +0200
Merge branch 'dev' into SP-1367
installer/cli/environments/pipeline-element | 2 -
installer/cli/environments/pipeline-element-nats | 2 -
pom.xml | 28 +-
streampipes-client-python/setup.py | 4 +-
.../apache/streampipes/client/api/PipelineApi.java | 5 +-
.../commons/configs/CouchDbConfigurations.java | 35 ---
.../commons/configs/CouchDbEnvKeys.java | 25 --
.../configs/DataExplorerConfigurations.java | 46 ---
.../commons/configs/DataExplorerEnvKeys.java | 28 --
.../dataexplorer/DataExplorerQueryManagement.java | 142 +++++++++
...ntV3.java => DataExplorerSchemaManagement.java} | 93 ++++--
.../dataexplorer/DataLakeManagementV4.java | 245 ---------------
.../api/IDataExplorerQueryManagement.java | 47 +++
.../IDataExplorerSchemaManagement.java} | 25 +-
.../IQueryStatement.java} | 14 +-
.../influx/DataExplorerInfluxQueryExecutor.java | 169 +++++++++++
.../DataLakeInfluxQueryBuilder.java} | 138 +++++----
.../DeleteQueryParams.java} | 45 ++-
.../param/ProvidedRestQueryParamConverter.java | 147 +++++++++
.../ProvidedRestQueryParams.java} | 12 +-
.../dataexplorer/param/SelectQueryParams.java | 140 +++++++++
.../SupportedRestQueryParams.java} | 4 +-
.../model/AggregationFunction.java} | 8 +-
.../model/FillClauseParams.java} | 20 +-
.../model/GroupByTagsClauseParams.java} | 24 +-
.../model/GroupByTimeClauseParams.java} | 19 +-
.../model/LimitClauseParams.java} | 19 +-
.../model/OffsetClauseParams.java} | 19 +-
.../model/OrderByClauseParams.java} | 20 +-
.../model/SelectClauseParams.java} | 73 ++---
.../{v4/params => param/model}/SelectColumn.java | 58 ++--
.../param/model/WhereClauseParams.java | 122 ++++++++
.../{v4 => query}/AutoAggregationHandler.java | 53 ++--
.../query/DataExplorerQueryExecutor.java | 106 +++++++
.../{v4 => }/query/QueryResultProvider.java | 28 +-
.../query/StreamedQueryResultProvider.java | 18 +-
.../query/writer/ConfiguredCsvOutputWriter.java | 10 +-
.../query/writer/ConfiguredJsonOutputWriter.java | 10 +-
.../query/writer/ConfiguredOutputWriter.java | 8 +-
.../{v4 => }/query/writer/OutputFormat.java | 2 +-
.../{v4 => }/query/writer/item/CsvItemWriter.java | 2 +-
.../{v4 => }/query/writer/item/ItemGenerator.java | 4 +-
.../{v4 => }/query/writer/item/JsonItemWriter.java | 2 +-
.../DataLakeQueryOrdering.java | 2 +-
.../FilterCondition.java} | 24 +-
.../querybuilder/IDataLakeQueryBuilder.java | 84 ++++++
.../dataexplorer/sdk/DataLakeQueryConstants.java | 30 --
.../dataexplorer/sdk/IDataLakeQueryBuilder.java | 74 -----
.../dataexplorer/{v4 => }/utils/TimeParser.java | 2 +-
.../v4/params/DeleteFromStatementParams.java | 31 --
.../dataexplorer/v4/params/FillParams.java | 35 ---
.../v4/params/WhereStatementParams.java | 122 --------
.../dataexplorer/v4/query/DataExplorerQueryV4.java | 234 --------------
.../dataexplorer/v4/query/QueryBuilder.java | 64 ----
.../v4/query/elements/DeleteFromStatement.java | 33 --
.../v4/query/elements/GroupingByTags.java | 43 ---
.../v4/query/elements/GroupingByTime.java | 34 ---
.../v4/query/elements/ItemLimitation.java | 34 ---
.../dataexplorer/v4/query/elements/Offset.java | 34 ---
.../v4/query/elements/OrderingByTime.java | 34 ---
.../v4/query/elements/SelectFromStatement.java | 51 ----
.../v4/query/elements/TimeBoundary.java | 40 ---
.../v4/query/elements/WhereStatement.java | 39 ---
.../dataexplorer/v4/template/QueryTemplatesV4.java | 63 ----
.../v4/utils/DataLakeManagementUtils.java | 163 ----------
.../dataexplorer/param/SelectQueryParamsTest.java | 208 +++++++++++++
.../param}/WhereStatementParamsTest.java | 27 +-
.../writer/TestConfiguredCsvOutputWriter.java | 7 +-
.../writer/TestConfiguredJsonOutputWriter.java | 7 +-
.../query/writer/TestConfiguredOutputWriter.java | 2 +-
.../query/writer/item/TestCsvItemWriter.java | 4 +-
.../query/writer/item/TestItemWriter.java | 2 +-
.../query/writer/item/TestJsonItemWriter.java | 4 +-
.../dataexplorer/sdk/DataLakeQueryBuilderTest.java | 6 +-
.../utils/ProvidedQueryParameterBuilder.java | 92 ++++++
.../streampipes-connect-adapters-iiot/pom.xml | 4 -
.../streampipes-connect-adapters/pom.xml | 4 -
.../sinks/brokers/jvm/BrokersJvmInit.java | 4 +-
.../sinks/brokers/jvm/kafka/KafkaController.java | 91 ------
.../sinks/brokers/jvm/kafka/KafkaParameters.java | 56 ++--
.../sinks/brokers/jvm/kafka/KafkaPublishSink.java | 130 ++++++++
.../sinks/brokers/jvm/kafka/KafkaPublisher.java | 86 ------
.../sinks/internal/jvm/SinksInternalJvmInit.java | 4 -
.../streampipes/ps/DataLakeMeasureResourceV3.java | 57 ----
.../streampipes/ps/DataLakeMeasureResourceV4.java | 23 +-
.../apache/streampipes/ps/DataLakeResourceV4.java | 70 +++--
.../apache/streampipes/rest/ResetManagement.java | 14 +-
streampipes-service-core/pom.xml | 1 -
.../service/core/StreamPipesResourceConfig.java | 2 -
ui/cypress/support/utils/PrepareTestDataUtils.ts | 8 +-
.../tests/datalake/configuration.smoke.spec.ts | 7 +-
ui/package-lock.json | 335 +++++++++++----------
ui/package.json | 8 +-
.../widgets/table/table-widget.component.html | 1 +
94 files changed, 2160 insertions(+), 2395 deletions(-)