You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2021/07/20 05:00:03 UTC
[incubator-streampipes] branch STREAMPIPES-319 updated (eb7c177 ->
2af797b)
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a change to branch STREAMPIPES-319
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git.
from eb7c177 [STREAMPIPES-319] Change auto-generated element id of generic adapters
add a3d1389 [STREAMPIPES-349] Add skeleton of the revised REST API
add d4e35f3 [STREAMPIPES-349] Add implementation of endpoint for 'getting all stored measurement series'
add a066b5d [STREAMPIPES-349] Rename method as well as list containing all stored measurement series
add 3e303a8 [STREAMPIPES-349] Add implementation of endpoint for 'removing all stored measurement series'
add 00767ef [STREAMPIPES-349] Add data lake configuration and retention policy defining classes
add b5bc9f8 [STREAMPIPES-349] Add label definition for data lake
add f3ced17 [STREAMPIPES-349] Add query defining classes for changes at the retention policies
add 3c30237 [STREAMPIPES-349] Update definition of configuration endpoints and add implementation
add f7cb766 [STREAMPIPES-349] Add first implementation of methods for getting and setting data lake configuration
add 3c3d74d [STREAMPIPES-349] Add abstract query element class and query parameter class
add de0279a [STREAMPIPES-349] Add query template class (includes database specific templates for various query statements)
add 84f16a6 [STREAMPIPES-349] Add query element 'Select From Statement' and corresponding parameter class
add a79e5eb [STREAMPIPES-349] Add query element 'Time Boundary' and corresponding parameter class
add 303b6e6 [STREAMPIPES-349] Add query element 'Ordering By Time' and corresponding parameter class
add 0268537 [STREAMPIPES-349] Add query element 'Grouping By Tags' and corresponding parameter class
add 12ed200 [STREAMPIPES-349] Add query element 'Grouping By Time' and corresponding parameter class
add 32e6b79 [STREAMPIPES-349] Add query element 'Item Limitation' and corresponding parameter class
add 756e42b [STREAMPIPES-349] Add query element 'Offset' and corresponding parameter class
add 5c0d435 [STREAMPIPES-349] Add query element 'Delete From Statement' and corresponding parameter class
add 6091bce [STREAMPIPES-349] Add Query Builder class
add b130424 [STREAMPIPES-349] Add Data Explorer Query class
add 9026b67 [STREAMPIPES-349] Add implementation of endpoint for removing data from a single measurement series
add 92cf0bf [STREAMPIPES-349] Add implementation of endpoint for getting data from data lake
add 2e87543 [STREAMPIPES-349] Add endpoint definition for downloading data from data lake (as csv- or json-file)
add cf1ea0b [STREAMPIPES-349] Add implementation of endpoint for downloading data from data lake
add fcf79b3 [STREAMPIPES-349] Add endpoint definition for dropping entire measurement series from data lake and removing related event property
add 0fbaef5 [STREAMPIPES-349] Add implementation of endpoint for removing entire measurement series and related event property
add da1e9c8 [STREAMPIPES-349] Add DataLakeResourceV4 to StreamPipesResourceConfig
add 89a48d3 [STREAMPIPES-349] Add DataLakeResourceV4 to openapi.yaml
add 4af58fc [STREAMPIPES-349] Refactoring of Query Params Extraction: enhanced decoupling of data lake management class and database-specific statements
add ef5be85 [STREAMPIPES-349] Refactor data retrieval and download endpoint to support selection of specified columns
add b57ac75 [hotfix] Remove unused imports
add bd2735c Merge pull request #45 from vesense/patch-2
add 9f18861 Merge branch 'dev' into STREAMPIPES-349
add ddb7ba4 Remove old code
new d901119 Merge branch 'STREAMPIPES-349' into STREAMPIPES-319
new 2af797b Merge branch 'STREAMPIPES-319' of github.com:apache/incubator-streampipes into STREAMPIPES-319
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../backend/StreamPipesResourceConfig.java | 153 ++++++------
.../src/main/resources/openapi.yaml | 1 +
.../dataexplorer/DataLakeManagementV4.java | 271 +++++++++++++++++++++
.../param/RetentionPolicyQueryParams.java | 20 ++
.../query/EditRetentionPolicyQuery.java | 56 +++++
.../query/ShowRetentionPolicyQuery.java | 38 +++
.../v4/params/DeleteFromStatementParams.java | 15 +-
.../dataexplorer/v4/params/FillParams.java | 19 +-
.../v4/params/GroupingByTagsParams.java | 33 +--
.../v4/params/GroupingByTimeParams.java | 20 +-
.../v4/params/ItemLimitationParams.java | 23 +-
.../dataexplorer/v4/params/OffsetParams.java | 33 ++-
.../v4/params/OrderingByTimeParams.java | 24 +-
.../dataexplorer/v4/params/QueryParamsV4.java | 24 +-
.../v4/params/SelectFromStatementParams.java | 57 +++++
.../dataexplorer/v4/params/TimeBoundaryParams.java | 23 +-
.../dataexplorer/v4/query/DataExplorerQueryV4.java | 137 +++++++++++
.../dataexplorer/v4/query/QueryBuilder.java | 50 ++--
.../v4/query/elements/DeleteFromStatement.java | 22 +-
.../v4/query/elements/FillStatement.java | 20 +-
.../v4/query/elements/GroupingByTags.java | 27 +-
.../v4/query/elements/GroupingByTime.java | 21 +-
.../v4/query/elements/ItemLimitation.java | 22 +-
.../dataexplorer/v4/query/elements/Offset.java | 24 +-
.../v4/query/elements/OrderingByTime.java | 21 +-
.../v4/query/elements/QueryElement.java | 17 +-
.../v4/query/elements/SelectFromStatement.java | 37 +++
.../v4/query/elements/TimeBoundary.java | 40 +++
.../dataexplorer/v4/template/QueryTemplatesV4.java | 79 ++++++
.../v4/utils/DataLakeManagementUtils.java | 91 +++++++
.../model/datalake/DataLakeConfiguration.java | 60 +++++
.../model/datalake/DataLakeRetentionPolicy.java | 62 +++++
.../{DataResult.java => LabelDefinition.java} | 34 ++-
.../apache/streampipes/ps/DataLakeResourceV4.java | 219 +++++++++++++++++
.../wrapper/flink/consumer/MqttFlinkConsumer.java | 2 -
.../datalake/datalake-rest.service.ts | 66 ++---
.../image-categorize/image-categorize.component.ts | 4 +-
ui/src/app/services/auth-status.service.ts | 2 +-
38 files changed, 1508 insertions(+), 359 deletions(-)
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/RetentionPolicyQueryParams.java
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/EditRetentionPolicyQuery.java
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/ShowRetentionPolicyQuery.java
copy streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/SpecificAdapterSetDescription.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/DeleteFromStatementParams.java (70%)
copy ui/src/app/sensors/components/general/options.controller.ts => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/FillParams.java (69%)
copy streampipes-connect/src/main/java/org/apache/streampipes/connect/GetNEvents.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/GroupingByTagsParams.java (55%)
copy streampipes-wrapper/src/main/java/org/apache/streampipes/wrapper/runtime/PipelineElement.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/GroupingByTimeParams.java (63%)
copy streampipes-model-client/src/main/java/org/apache/streampipes/model/client/monitoring/pipeline/BurrowTopicInfo.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/ItemLimitationParams.java (63%)
copy ui/src/app/files/components/file-overview/file-overview.component.scss => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/OffsetParams.java (64%)
copy streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/guess/PropertyGuessResults.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/OrderingByTimeParams.java (62%)
copy ui/src/app/sensors/components/general/advanced-settings.controller.ts => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/QueryParamsV4.java (70%)
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/SelectFromStatementParams.java
copy ui/src/app/core-model/datalake/DateRange.ts => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/TimeBoundaryParams.java (59%)
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/DataExplorerQueryV4.java
copy streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/PeConfig.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/QueryBuilder.java (50%)
copy streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/AddTimestampPipelineElement.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/DeleteFromStatement.java (58%)
copy streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/IPipelineElementConnectionStorage.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/FillStatement.java (72%)
copy streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpKvManagement.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/GroupingByTags.java (53%)
copy streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/AddTimestampPipelineElement.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/GroupingByTime.java (60%)
copy streampipes-backend/src/main/java/org/apache/streampipes/backend/WelcomePageController.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/ItemLimitation.java (60%)
copy streampipes-model-client/src/main/java/org/apache/streampipes/model/client/monitoring/pipeline/BurrowTopicInfo.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/Offset.java (63%)
copy streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/AddTimestampPipelineElement.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/OrderingByTime.java (60%)
copy streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/SpecificAdapterSetDescription.java => streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/QueryElement.java (67%)
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/SelectFromStatement.java
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/TimeBoundary.java
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/template/QueryTemplatesV4.java
create mode 100644 streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/utils/DataLakeManagementUtils.java
create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeConfiguration.java
create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeRetentionPolicy.java
copy streampipes-model/src/main/java/org/apache/streampipes/model/datalake/{DataResult.java => LabelDefinition.java} (61%)
create mode 100644 streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
[incubator-streampipes] 01/02: Merge branch 'STREAMPIPES-349' into
STREAMPIPES-319
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch STREAMPIPES-319
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit d9011190739ff61c1b095af6d066d0f5221e59e1
Merge: b179928 ddb7ba4
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Tue Jul 20 06:59:37 2021 +0200
Merge branch 'STREAMPIPES-349' into STREAMPIPES-319
.../backend/StreamPipesResourceConfig.java | 153 ++++++------
.../src/main/resources/openapi.yaml | 1 +
.../dataexplorer/DataLakeManagementV4.java | 271 +++++++++++++++++++++
.../param/RetentionPolicyQueryParams.java | 20 ++
.../query/EditRetentionPolicyQuery.java | 56 +++++
.../query/ShowRetentionPolicyQuery.java | 38 +++
.../v4/params/DeleteFromStatementParams.java | 18 +-
.../dataexplorer/v4/params/FillParams.java | 23 +-
.../v4/params/GroupingByTagsParams.java | 29 ++-
.../v4/params/GroupingByTimeParams.java | 23 +-
.../v4/params/ItemLimitationParams.java | 24 +-
.../dataexplorer/v4/params/OffsetParams.java | 24 +-
.../v4/params/OrderingByTimeParams.java | 23 +-
.../dataexplorer/v4/params/QueryParamsV4.java | 23 +-
.../v4/params/SelectFromStatementParams.java | 57 +++++
.../dataexplorer/v4/params/TimeBoundaryParams.java | 30 ++-
.../dataexplorer/v4/query/DataExplorerQueryV4.java | 137 +++++++++++
.../dataexplorer/v4/query/QueryBuilder.java | 54 ++++
.../v4/query/elements/DeleteFromStatement.java | 22 +-
.../v4/query/elements/FillStatement.java | 21 +-
.../v4/query/elements/GroupingByTags.java | 30 ++-
.../v4/query/elements/GroupingByTime.java | 21 +-
.../v4/query/elements/ItemLimitation.java | 21 +-
.../dataexplorer/v4/query/elements/Offset.java | 21 +-
.../v4/query/elements/OrderingByTime.java | 21 +-
.../v4/query/elements/QueryElement.java | 22 +-
.../v4/query/elements/SelectFromStatement.java | 37 +++
.../v4/query/elements/TimeBoundary.java | 40 +++
.../dataexplorer/v4/template/QueryTemplatesV4.java | 79 ++++++
.../v4/utils/DataLakeManagementUtils.java | 91 +++++++
.../model/datalake/DataLakeConfiguration.java | 60 +++++
.../model/datalake/DataLakeRetentionPolicy.java | 62 +++++
.../model/datalake/LabelDefinition.java | 33 ++-
.../apache/streampipes/ps/DataLakeResourceV4.java | 219 +++++++++++++++++
.../wrapper/flink/consumer/MqttFlinkConsumer.java | 2 -
.../datalake/datalake-rest.service.ts | 66 ++---
.../image-categorize/image-categorize.component.ts | 4 +-
ui/src/app/services/auth-status.service.ts | 2 +-
38 files changed, 1562 insertions(+), 316 deletions(-)
diff --cc streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesResourceConfig.java
index 1c9f03b,0cf6a43..e21f900
--- a/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesResourceConfig.java
+++ b/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesResourceConfig.java
@@@ -46,86 -48,85 +47,86 @@@ import javax.ws.rs.ApplicationPath
@ApplicationPath("/api")
public class StreamPipesResourceConfig extends ResourceConfig {
- public StreamPipesResourceConfig() {
- register(Authentication.class);
- register(AssetDashboard.class);
- register(AutoComplete.class);
- register(CategoryResource.class);
- register(ConsulConfig.class);
- register(ContainerProvidedOptions.class);
- register(Couchdb.class);
- register(DashboardWidget.class);
- register(Dashboard.class);
- register(DataLakeResourceV3.class);
- register(DataLakeNoUserResourceV3.class);
- register(DataStream.class);
- register(Deployment.class);
- register(FileServingResource.class);
- register(InternalPipelineTemplates.class);
- register(LabelResource.class);
- register(MeasurementUnitResource.class);
- register(Notification.class);
- register(OntologyMeasurementUnit.class);
- register(OntologyPipelineElement.class);
- register(PipelineCanvasMetadataCache.class);
- register(PipelineCanvasMetadataResource.class);
- register(PipelineCache.class);
- register(PipelineCategory.class);
- register(PipelineElementAsset.class);
- register(PipelineElementCategory.class);
- register(PipelineElementFile.class);
- register(PipelineElementImportNoUser.class);
- register(PipelineElementImport.class);
- register(PipelineElementPreview.class);
- register(PipelineElementRuntimeInfo.class);
- register(PipelineMonitoring.class);
- register(PipelineNoUserResource.class);
- register(PipelineTemplate.class);
- register(PipelineResource.class);
- register(ExtensionsServiceEndpointResource.class);
- register(SemanticEventConsumer.class);
- register(SemanticEventProcessingAgent.class);
- register(SemanticEventProducer.class);
- register(Setup.class);
- register(User.class);
- register(Version.class);
- register(PipelineElementAsset.class);
- register(DataLakeDashboardResource.class);
- register(DataLakeWidgetResource.class);
- register(DataLakeResourceV3.class);
- register(DataLakeNoUserResourceV3.class);
- register(PipelineElementFile.class);
- register(FileServingResource.class);
- register(DashboardWidget.class);
- register(Dashboard.class);
- register(VirtualSensor.class);
- register(Visualization.class);
- register(VisualizablePipeline.class);
+ public StreamPipesResourceConfig() {
+ register(Authentication.class);
+ register(AssetDashboard.class);
+ register(AutoComplete.class);
+ register(CategoryResource.class);
+ register(ConsulConfig.class);
+ register(ContainerProvidedOptions.class);
+ register(Couchdb.class);
+ register(DashboardWidget.class);
+ register(Dashboard.class);
+ register(DataLakeResourceV3.class);
+ register(DataLakeNoUserResourceV3.class);
+ register(DataStream.class);
+ register(Deployment.class);
+ register(FileServingResource.class);
+ register(InternalPipelineTemplates.class);
+ register(LabelResource.class);
+ register(MeasurementUnitResource.class);
+ register(Notification.class);
+ register(OntologyMeasurementUnit.class);
+ register(OntologyPipelineElement.class);
+ register(PipelineCanvasMetadataCache.class);
+ register(PipelineCanvasMetadataResource.class);
+ register(PipelineCache.class);
+ register(PipelineCategory.class);
+ register(PipelineElementAsset.class);
+ register(PipelineElementCategory.class);
+ register(PipelineElementFile.class);
+ register(PipelineElementImportNoUser.class);
+ register(PipelineElementImport.class);
+ register(PipelineElementPreview.class);
+ register(PipelineElementRuntimeInfo.class);
+ register(PipelineMonitoring.class);
+ register(PipelineNoUserResource.class);
+ register(PipelineTemplate.class);
+ register(PipelineResource.class);
- register(RdfEndpoint.class);
++ register(ExtensionsServiceEndpointResource.class);
+ register(SemanticEventConsumer.class);
+ register(SemanticEventProcessingAgent.class);
+ register(SemanticEventProducer.class);
+ register(Setup.class);
+ register(User.class);
+ register(Version.class);
++ register(PipelineElementAsset.class);
+ register(DataLakeDashboardResource.class);
+ register(DataLakeWidgetResource.class);
++ register(DataLakeResourceV3.class);
+ register(DataLakeNoUserResourceV3.class);
++ register(PipelineElementFile.class);
+ register(FileServingResource.class);
+ register(DashboardWidget.class);
+ register(Dashboard.class);
+ register(VirtualSensor.class);
+ register(Visualization.class);
+ register(VisualizablePipeline.class);
- // Serializers
- register(GsonWithIdProvider.class);
- register(GsonWithoutIdProvider.class);
- register(GsonClientModelProvider.class);
- register(JacksonSerializationProvider.class);
- register(MultiPartFeature.class);
-
- // Platform Services
- register(PipelineElementTemplateResource.class);
+ // Serializers
+ register(GsonWithIdProvider.class);
+ register(GsonWithoutIdProvider.class);
+ register(GsonClientModelProvider.class);
- register(JsonLdProvider.class);
+ register(JacksonSerializationProvider.class);
+ register(MultiPartFeature.class);
- register(OpenApiResource.class);
+ // Platform Services
+ register(PipelineElementTemplateResource.class);
+ register(DataLakeResourceV4.class);
-
+ register(OpenApiResource.class);
- // Connect Master
- register(WelcomePageMaster.class);
- register(AdapterResource.class);
- register(AdapterTemplateResource.class);
- register(DescriptionResource.class);
- register(SourcesResource.class);
- register(GuessResource.class);
+ // Connect Master
+ register(WelcomePageMaster.class);
+ register(AdapterResource.class);
+ register(AdapterTemplateResource.class);
+ register(DescriptionResource.class);
+ register(SourcesResource.class);
+ register(GuessResource.class);
// register(MultiPartFeature.class);
- register(UnitResource.class);
- register(WorkerAdministrationResource.class);
- register(RuntimeResolvableResource.class);
- }
+ register(UnitResource.class);
+ register(WorkerAdministrationResource.class);
+ register(RuntimeResolvableResource.class);
+ }
}
diff --cc streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/FillParams.java
index 65d01d7,63f3ac3..b415d83
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/FillParams.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/params/FillParams.java
@@@ -16,20 -16,17 +16,19 @@@
*
*/
- import {Injectable} from '@angular/core';
-package org.apache.streampipes.connect.adapter.exception;
++package org.apache.streampipes.dataexplorer.v4.params;
- @Injectable()
- export class AuthStatusService {
-public class AdapterException extends Exception {
- public AdapterException() {}
++public class FillParams extends QueryParamsV4 {
++ String fill = "fill(none)";
- user: any;
- email: string;
- username: string;
- token: string;
- authenticated: boolean = false;
- configured: boolean = false;
- darkMode: boolean = true;
-
- constructor() {
- public AdapterException(String message)
- {
- super(message);
++ public static FillParams from(String measurementID) {
++ return new FillParams(measurementID);
++ }
++ protected FillParams(String index) {
++ super(index);
}
- public AdapterException(String message, Throwable cause) {
- super(message, cause);
++ public String getFill() {
++ return fill;
+ }
}
diff --cc streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/DataExplorerQueryV4.java
index 0000000,9db9be6..6c4b64c
mode 000000,100644..100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/DataExplorerQueryV4.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/DataExplorerQueryV4.java
@@@ -1,0 -1,122 +1,137 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+ package org.apache.streampipes.dataexplorer.v4.query;
+
+ import org.apache.streampipes.config.backend.BackendConfig;
+ import org.apache.streampipes.dataexplorer.utils.DataExplorerUtils;
+ import org.apache.streampipes.dataexplorer.v4.params.*;
+ import org.apache.streampipes.dataexplorer.v4.query.elements.*;
+ import org.apache.streampipes.dataexplorer.v4.utils.DataLakeManagementUtils;
+ import org.apache.streampipes.model.datalake.DataResult;
+ import org.influxdb.InfluxDB;
+ import org.influxdb.dto.Query;
+ import org.influxdb.dto.QueryResult;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.Map;
+
+ public class DataExplorerQueryV4 {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DataExplorerQueryV4.class);
+
+
+ protected Map<String, QueryParamsV4> params;
+
+ public DataExplorerQueryV4(Map<String, QueryParamsV4> params) {
+ this.params = params;
+ }
+
+ public DataResult executeQuery() throws RuntimeException {
+ InfluxDB influxDB = DataExplorerUtils.getInfluxDBClient();
+ List<QueryElement> queryElements = getQueryElements();
+
+ QueryBuilder queryBuilder = QueryBuilder.create(BackendConfig.INSTANCE.getInfluxDatabaseName());
+ Query query = queryBuilder.build(queryElements);
+ LOG.debug("Data Lake Query (database:" + query.getDatabase() + "): " + query.getCommand());
+
+ QueryResult result = influxDB.query(query);
+ LOG.debug("Data Lake Query Result: " + result.toString());
+ DataResult dataResult = postQuery(result);
+
+ influxDB.close();
+ return dataResult;
+ }
+
+ protected DataResult convertResult(QueryResult result) {
+ if (result.getResults().get(0).getSeries() != null) {
+ return convertResult(result.getResults().get(0).getSeries().get(0));
+ } else {
+ return new DataResult();
+ }
+ }
+
+ protected DataResult convertResult(QueryResult.Series series) {
+ List<String> columns = series.getColumns();
+ for (int i = 0; i < columns.size(); i++) {
+ String replacedColumnName = columns.get(i).replaceAll("mean_", "");
+ columns.set(i, replacedColumnName);
+ }
+ List values = series.getValues();
- return new DataResult(values.size(), columns, values);
++
++ List resultingValues = new ArrayList();
++
++ values.forEach(v -> {
++
++ if (!((List<String>)v).contains(null)) {
++ resultingValues.add(v);
++ }
++ });
++
++ return new DataResult(values.size(), columns, resultingValues);
+ }
+
+ protected DataResult postQuery(QueryResult result) throws RuntimeException {
+ return convertResult(result);
+ }
+
+ protected List<QueryElement> getQueryElements() {
+ List<QueryElement> queryElements = new ArrayList<>();
+
+ if (this.params.containsKey(DataLakeManagementUtils.SELECT_FROM)) {
+ queryElements.add(new SelectFromStatement((SelectFromStatementParams) this.params.get(DataLakeManagementUtils.SELECT_FROM)));
+ } else {
+ queryElements.add(new DeleteFromStatement((DeleteFromStatementParams) this.params.get(DataLakeManagementUtils.SELECT_FROM)));
+ }
+
+ if (this.params.containsKey(DataLakeManagementUtils.WHERE)) {
+ queryElements.add(new TimeBoundary((TimeBoundaryParams) this.params.get(DataLakeManagementUtils.WHERE)));
+ }
+
+ if (this.params.containsKey(DataLakeManagementUtils.GROUP_BY_TIME)) {
+ queryElements.add(new GroupingByTime((GroupingByTimeParams) this.params.get(DataLakeManagementUtils.GROUP_BY_TIME)));
+
+ } else if (this.params.containsKey(DataLakeManagementUtils.GROUP_BY_TAGS)) {
+ queryElements.add(new GroupingByTags((GroupingByTagsParams) this.params.get(DataLakeManagementUtils.GROUP_BY_TAGS)));
+ }
+
++ if (this.params.containsKey(DataLakeManagementUtils.FILL)) {
++ queryElements.add(new FillStatement((FillParams) this.params.get(DataLakeManagementUtils.FILL)));
++ }
++
+ if (this.params.containsKey(DataLakeManagementUtils.ORDER_DESCENDING)) {
+ queryElements.add(new OrderingByTime((OrderingByTimeParams) this.params.get(DataLakeManagementUtils.ORDER_DESCENDING)));
+ } else if (this.params.containsKey(DataLakeManagementUtils.SELECT_FROM)) {
+ queryElements.add(new OrderingByTime(OrderingByTimeParams.from(this.params.get(DataLakeManagementUtils.SELECT_FROM).getIndex(), "ASC")));
+ }
+
+ if (this.params.containsKey(DataLakeManagementUtils.LIMIT)) {
+ queryElements.add(new ItemLimitation((ItemLimitationParams) this.params.get(DataLakeManagementUtils.LIMIT)));
+ }
+
+ if (this.params.containsKey(DataLakeManagementUtils.OFFSET)) {
+ queryElements.add(new Offset((OffsetParams) this.params.get(DataLakeManagementUtils.OFFSET)));
+ }
+
++
+ return queryElements;
+ }
+ }
diff --cc streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/FillStatement.java
index 65d01d7,63f3ac3..a4f156a
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/FillStatement.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/query/elements/FillStatement.java
@@@ -16,20 -16,17 +16,17 @@@
*
*/
- import {Injectable} from '@angular/core';
-package org.apache.streampipes.connect.adapter.exception;
++package org.apache.streampipes.dataexplorer.v4.query.elements;
- @Injectable()
- export class AuthStatusService {
-public class AdapterException extends Exception {
- public AdapterException() {}
++import org.apache.streampipes.dataexplorer.v4.params.FillParams;
- user: any;
- email: string;
- username: string;
- token: string;
- authenticated: boolean = false;
- configured: boolean = false;
- darkMode: boolean = true;
-
- constructor() {
- public AdapterException(String message)
- {
- super(message);
++public class FillStatement extends QueryElement<FillParams> {
++ public FillStatement(FillParams fillParams) {
++ super(fillParams);
}
- public AdapterException(String message, Throwable cause) {
- super(message, cause);
++ @Override
++ protected String buildStatement(FillParams fillParams) {
++ return fillParams.getFill();
+ }
}
diff --cc streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/utils/DataLakeManagementUtils.java
index 0000000,3726253..a82abb0
mode 000000,100644..100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/utils/DataLakeManagementUtils.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/utils/DataLakeManagementUtils.java
@@@ -1,0 -1,87 +1,91 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+ package org.apache.streampipes.dataexplorer.v4.utils;
+
+ import org.apache.streampipes.dataexplorer.v4.params.*;
+
+ import java.util.HashMap;
+ import java.util.Map;
+
+ public class DataLakeManagementUtils {
+
+ public static final String SELECT_FROM = "SELECT";
+ public static final String WHERE = "WHERE";
+ public static final String GROUP_BY_TAGS = "GROUPBY";
+ public static final String GROUP_BY_TIME = "GROUPBYTIME";
+ public static final String ORDER_DESCENDING = "DESC";
+ public static final String LIMIT = "LIMIT";
+ public static final String OFFSET = "OFFSET";
++ public static final String FILL = "FILL";
+
+ public static final String DELETE_FROM = "DELETE";
+
+ public static Map<String, QueryParamsV4> getSelectQueryParams(String measurementID, String columns, Long startDate, Long endDate, Integer page, Integer limit, Integer offset, String groupBy, String order, String aggregationFunction, String timeInterval) {
+ Map<String, QueryParamsV4> queryParts = new HashMap<>();
+
+ queryParts.put(SELECT_FROM, SelectFromStatementParams.from(measurementID, columns, aggregationFunction));
+
+ if (startDate != null || endDate != null) {
+ queryParts.put(WHERE, TimeBoundaryParams.from(measurementID, startDate, endDate));
+ }
+
+
+ if (timeInterval != null && aggregationFunction != null) {
+ if (groupBy == null) {
+ queryParts.put(GROUP_BY_TIME, GroupingByTimeParams.from(measurementID, timeInterval));
+ } else {
+ groupBy = groupBy + ",time(" + timeInterval + ")";
+ }
+ }
+
+ if (groupBy != null) {
+ queryParts.put(GROUP_BY_TAGS, GroupingByTagsParams.from(measurementID, groupBy));
+ }
+
++ queryParts.put(FILL, FillParams.from(measurementID));
++
+ if (order != null) {
+ if (order.equals(ORDER_DESCENDING)) {
+ queryParts.put(ORDER_DESCENDING, OrderingByTimeParams.from(measurementID, order));
+ }
+ }
+
+ if (limit != null) {
+ queryParts.put(LIMIT, ItemLimitationParams.from(measurementID, limit));
+ }
+
+ if (offset != null) {
+ queryParts.put(OFFSET, OffsetParams.from(measurementID, offset));
+ } else if (limit != null && page != null) {
+ queryParts.put(OFFSET, OffsetParams.from(measurementID, page * limit));
+ }
+
++
+ return queryParts;
+ }
+
+ public static Map<String, QueryParamsV4> getDeleteQueryParams(String measurementID, Long startDate, Long endDate) {
+ Map<String, QueryParamsV4> queryParts = new HashMap<>();
+ queryParts.put(DELETE_FROM, DeleteFromStatementParams.from(measurementID));
+ if (startDate != null || endDate != null) {
+ queryParts.put(WHERE, TimeBoundaryParams.from(measurementID, startDate, endDate));
+ }
+ return queryParts;
+ }
+ }
diff --cc ui/src/app/core-services/datalake/datalake-rest.service.ts
index fee644f,1326669..d5a6222
--- a/ui/src/app/core-services/datalake/datalake-rest.service.ts
+++ b/ui/src/app/core-services/datalake/datalake-rest.service.ts
@@@ -41,7 -41,12 +41,17 @@@ export class DatalakeRestService
return this.baseUrl + '/api/v3/users/' + this.authStatusService.email + '/datalake';
}
- getAllInfos(): Observable<DataLakeMeasure[]> {
+ private get dataLakeUrlV4() {
+ return this.baseUrl + '/api/v4/users/' + this.authStatusService.email + '/datalake';
+ }
+
+
++ public getD(): Observable<DataLakeMeasure[]> {
++
++ return null;
++ }
++
+ getAllInfos(): Observable<DataLakeMeasure[]> {
return this.http.get(this.dataLakeUrlV3 + '/info').pipe(map(response => {
return (response as any[]).map(p => DataLakeMeasure.fromData(p));
}));
@@@ -56,7 -61,8 +66,25 @@@
}
getData(index, startDate, endDate, aggregationTimeUnit, aggregationValue): Observable<DataResult> {
- return this.http.get<DataResult>(this.dataLakeUrlV3 + '/data/' + index + '/' + startDate + '/' + endDate + '?aggregationUnit=' + aggregationTimeUnit + '&aggregationValue=' + aggregationValue);
- return this.http.get<DataResult>(
- this.dataLakeUrlV3 + '/data/' + index + '/' + startDate + '/' + endDate + '?aggregationUnit=' + aggregationTimeUnit + '&aggregationValue=' + aggregationValue);
++ // return this.http.get<DataResult>(
++ // this.dataLakeUrlV3 + '/data/' + index + '/' + startDate + '/' + endDate + '?aggregationUnit=' + aggregationTimeUnit + '&aggregationValue=' + aggregationValue);
++
++ const url = this.dataLakeUrlV4 + '/measurements/' + index;
++
++ return this.http.get<PageResult>(url, {
++ params: {
++ // columns: 'id1234',
++ startDate: startDate,
++ endDate: endDate,
++ // page: 0,
++ // limit: 1,
++ // offset: 1,
++ // groupBy: 'asd,sadf',
++ // order: 'ASC',
++ aggregationFunction: 'mean',
++ timeInterval: '1s'
++ }
++ });
}
getGroupedData(index, startDate, endDate, aggregationTimeUnit, aggregationValue, groupingTag) {
@@@ -67,10 -75,11 +97,6 @@@
return this.http.get<DataResult>(this.dataLakeUrlV3 + '/data/' + index + '/' + startDate + '/' + endDate);
}
-- getGroupedDataAutoAggergation(index, startDate, endDate, groupingTag) {
- return this.http.get<GroupedDataResult>(this.dataLakeUrlV3 + '/data/' + index + '/' + startDate + '/' + endDate + '/grouping/' + groupingTag);
- return this.http.get<GroupedDataResult>(
- this.dataLakeUrlV3 + '/data/' + index + '/' + startDate + '/' + endDate + '/grouping/' + groupingTag);
-- }
--
downloadRowData(index, format) {
const request = new HttpRequest('GET', this.dataLakeUrlV3 + '/data/' + index + '/download?format=' + format, {
reportProgress: true,
diff --cc ui/src/app/services/auth-status.service.ts
index 65d01d7,c8eb371..760d39e
--- a/ui/src/app/services/auth-status.service.ts
+++ b/ui/src/app/services/auth-status.service.ts
@@@ -27,7 -27,6 +27,7 @@@ export class AuthStatusService
token: string;
authenticated: boolean = false;
configured: boolean = false;
- darkMode: boolean = true;
++ darkMode: boolean = false;
constructor() {
}
[incubator-streampipes] 02/02: Merge branch 'STREAMPIPES-319' of
github.com:apache/incubator-streampipes into STREAMPIPES-319
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch STREAMPIPES-319
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 2af797b211e4feac6f1a0f56aa8e58a92a0289ea
Merge: d901119 eb7c177
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Tue Jul 20 06:59:48 2021 +0200
Merge branch 'STREAMPIPES-319' of github.com:apache/incubator-streampipes into STREAMPIPES-319
streampipes-backend/src/main/resources/shiro.ini | 2 +-
.../constants/GlobalStreamPipesConstants.java | 2 +
.../master/management/SourcesManagement.java | 14 +++---
.../container/master/rest/SourcesResource.java | 9 ++--
.../connect/container/master/util/Utils.java | 4 +-
.../master/management/SourcesManagementTest.java | 2 +-
.../apache/streampipes/model/client/user/User.java | 9 ++++
.../org/apache/streampipes/model/SpDataSet.java | 9 ++++
.../adapter/GenericAdapterSetDescription.java | 4 +-
.../adapter/GenericAdapterStreamDescription.java | 3 +-
.../streampipes/model/util/ElementIdGenerator.java | 12 +++++-
.../ExtensionsServiceEndpointGenerator.java | 2 +-
.../manager/execution/http/GraphSubmitter.java | 16 +++++--
.../manager/execution/http/PipelineExecutor.java | 50 ++++++++++++++++++----
.../manager/setup/CouchDbInstallationStep.java | 2 +-
.../org/apache/streampipes/rest/impl/User.java | 16 +++++++
.../serializers/json/GsonSerializer.java | 2 +
.../connect/services/data-marketplace.service.ts | 2 -
.../app/core-model/gen/streampipes-model-client.ts | 4 +-
.../core/components/iconbar/iconbar.component.html | 17 +++++---
.../components/iconbar/iconbar.component.scss} | 6 +++
.../core/components/iconbar/iconbar.component.ts | 4 +-
.../streampipes/streampipes.component.html | 6 +--
.../streampipes/streampipes.component.scss | 4 ++
.../core/components/toolbar/toolbar.component.html | 19 ++++----
.../core/components/toolbar/toolbar.component.scss | 15 +++++++
.../core/components/toolbar/toolbar.component.ts | 15 +++++--
.../data-explorer-dashboard-grid.component.scss | 12 +++---
...data-explorer-dashboard-overview.component.html | 4 +-
...data-explorer-dashboard-overview.component.scss | 11 ++---
.../time-selector/timeRangeSelector.component.html | 12 +++---
.../time-selector/timeRangeSelector.component.scss | 8 ++--
.../data-explorer-dashboard-widget.component.scss | 6 ++-
.../widgets/table/table-widget.component.html | 2 +-
.../widgets/table/table-widget.component.scss | 3 +-
.../no-data/no-data-in-date-range.component.css | 11 +++++
.../no-data/no-data-in-date-range.component.html | 5 ++-
.../app/data-explorer/data-explorer.component.css | 2 -
.../app/data-explorer/data-explorer.component.html | 8 ++--
...explorer-add-visualization-dialog.component.css | 10 ++---
...xplorer-add-visualization-dialog.component.html | 2 +-
ui/src/app/info/about/about.component.html | 17 +++-----
ui/src/app/info/info.component.html | 8 ++--
.../login/components/setup/setup.component.html | 4 +-
.../general-profile-settings.component.html | 21 +++++++++
.../general-profile-settings.component.scss | 10 +++++
.../general/general-profile-settings.component.ts | 27 +++++++++++-
ui/src/app/profile/profile.service.ts | 6 +++
ui/src/scss/sp/buttons.ng1.scss | 1 +
ui/src/scss/sp/layout.scss | 8 ++++
ui/src/scss/sp/main.scss | 4 +-
ui/src/scss/sp/sp-theme.scss | 4 ++
ui/src/scss/sp/widgets.scss | 4 +-
53 files changed, 330 insertions(+), 130 deletions(-)