You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2022/08/11 19:54:03 UTC
[incubator-streampipes] branch dev updated (2795d664e -> 9db5ad788)
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
from 2795d664e Merge branch 'dev' of github.com:apache/incubator-streampipes into dev
new 139134966 Merge branch 'dev' of github.com:apache/incubator-streampipes into dev
new a1f653e42 [hotfix] Reduce default page height
new 16f97b68a [STREAMPIPES-573] Add delimiter selection to data download dialog
new 9db5ad788 Merge branch 'dev' of github.com:apache/incubator-streampipes into dev
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../dataexplorer/DataLakeManagementV4.java | 15 ++++++++++----
.../v4/SupportedDataLakeQueryParameters.java | 2 ++
.../apache/streampipes/ps/DataLakeResourceV4.java | 1 +
.../src/lib/apis/datalake-rest.service.ts | 14 +++++++++----
.../basic-nav-tabs/basic-nav-tabs.component.scss | 2 +-
.../data-download-dialog.component.html | 24 ++++++++++++++--------
.../data-download-dialog.component.scss | 4 ++--
.../data-download-dialog.component.ts | 22 ++++++++++++++------
.../pipeline-element-icon-stand.component.scss | 2 +-
ui/src/scss/sp/main.scss | 2 +-
10 files changed, 61 insertions(+), 27 deletions(-)
[incubator-streampipes] 04/04: Merge branch 'dev' of github.com:apache/incubator-streampipes into dev
Posted by ri...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 9db5ad788c58e84795428b3243eb16eae03b36f3
Merge: 16f97b68a 2795d664e
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Thu Aug 11 21:53:53 2022 +0200
Merge branch 'dev' of github.com:apache/incubator-streampipes into dev
[incubator-streampipes] 02/04: [hotfix] Reduce default page height
Posted by ri...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit a1f653e428779c6382d84fc1c808da708562d54c
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Mon Aug 8 20:05:54 2022 +0200
[hotfix] Reduce default page height
---
.../src/lib/components/basic-nav-tabs/basic-nav-tabs.component.scss | 2 +-
.../pipeline-element-icon-stand.component.scss | 2 +-
ui/src/scss/sp/main.scss | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/ui/projects/streampipes/shared-ui/src/lib/components/basic-nav-tabs/basic-nav-tabs.component.scss b/ui/projects/streampipes/shared-ui/src/lib/components/basic-nav-tabs/basic-nav-tabs.component.scss
index 19c8ca2a3..4bbebfcc0 100644
--- a/ui/projects/streampipes/shared-ui/src/lib/components/basic-nav-tabs/basic-nav-tabs.component.scss
+++ b/ui/projects/streampipes/shared-ui/src/lib/components/basic-nav-tabs/basic-nav-tabs.component.scss
@@ -31,7 +31,7 @@
.page-container {
margin: 10px;
border: 1px solid var(--color-bg-3);
- min-height: calc(100% - 20px);
+ min-height: calc(100% - 50px);
}
.page-container-padding-inner {
diff --git a/ui/src/app/editor/components/pipeline-element-icon-stand/pipeline-element-icon-stand.component.scss b/ui/src/app/editor/components/pipeline-element-icon-stand/pipeline-element-icon-stand.component.scss
index fcd2a84c3..be1bab152 100644
--- a/ui/src/app/editor/components/pipeline-element-icon-stand/pipeline-element-icon-stand.component.scss
+++ b/ui/src/app/editor/components/pipeline-element-icon-stand/pipeline-element-icon-stand.component.scss
@@ -25,7 +25,7 @@
}
.icon-stand {
- max-height: calc(100vh - 118px);
+ max-height: calc(100vh - 148px);
overflow-y: auto;
background: var(--color-bg-1);
margin-top: 5px;
diff --git a/ui/src/scss/sp/main.scss b/ui/src/scss/sp/main.scss
index ac216e040..a934bee79 100644
--- a/ui/src/scss/sp/main.scss
+++ b/ui/src/scss/sp/main.scss
@@ -367,7 +367,7 @@ md-content {
.page-container {
margin: 10px;
border: 1px solid var(--color-bg-3);
- min-height: calc(100% - 20px);
+ min-height: calc(100% - 60px);
//height: 100%;
}
[incubator-streampipes] 01/04: Merge branch 'dev' of github.com:apache/incubator-streampipes into dev
Posted by ri...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 139134966ae70f892202763a4312e2e8e60631fe
Merge: f08476979 e65c67350
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Mon Aug 8 16:14:48 2022 +0200
Merge branch 'dev' of github.com:apache/incubator-streampipes into dev
.../connect/iiot/adapters/opcua/OpcUaAdapter.java | 367 +++++++++++----------
.../iiot/adapters/opcua/utils/OpcUaUtil.java | 41 ++-
ui/cypress/tests/adapter/opcAdapter.ts | 2 +-
3 files changed, 222 insertions(+), 188 deletions(-)
[incubator-streampipes] 03/04: [STREAMPIPES-573] Add delimiter selection to data download dialog
Posted by ri...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 16f97b68a3b423660a173f88f98fc4ba4dc8d7ec
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Thu Aug 11 21:53:43 2022 +0200
[STREAMPIPES-573] Add delimiter selection to data download dialog
---
.../dataexplorer/DataLakeManagementV4.java | 15 ++++++++++----
.../v4/SupportedDataLakeQueryParameters.java | 2 ++
.../apache/streampipes/ps/DataLakeResourceV4.java | 1 +
.../src/lib/apis/datalake-rest.service.ts | 14 +++++++++----
.../data-download-dialog.component.html | 24 ++++++++++++++--------
.../data-download-dialog.component.scss | 4 ++--
.../data-download-dialog.component.ts | 22 ++++++++++++++------
7 files changed, 58 insertions(+), 24 deletions(-)
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java
index 523cbe8aa..8f6a858ff 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java
@@ -36,7 +36,10 @@ import org.apache.streampipes.model.datalake.DataLakeConfiguration;
import org.apache.streampipes.model.datalake.DataLakeMeasure;
import org.apache.streampipes.model.datalake.DataLakeRetentionPolicy;
import org.apache.streampipes.model.datalake.SpQueryResult;
-import org.apache.streampipes.model.schema.*;
+import org.apache.streampipes.model.schema.EventProperty;
+import org.apache.streampipes.model.schema.EventPropertyList;
+import org.apache.streampipes.model.schema.EventPropertyNested;
+import org.apache.streampipes.model.schema.EventPropertyPrimitive;
import org.apache.streampipes.storage.api.IDataLakeStorage;
import org.apache.streampipes.storage.couchdb.utils.Utils;
import org.apache.streampipes.storage.management.StorageDispatcher;
@@ -45,7 +48,6 @@ import org.influxdb.dto.Query;
import org.influxdb.dto.QueryResult;
import org.lightcouch.CouchDbClient;
-import javax.xml.crypto.Data;
import java.io.IOException;
import java.io.OutputStream;
import java.time.Instant;
@@ -149,6 +151,11 @@ public class DataLakeManagementV4 {
}
boolean isFirstDataObject = true;
+ String delimiter = ",";
+
+ if (params.has(QP_CSV_DELIMITER)) {
+ delimiter = params.getAsString(QP_CSV_DELIMITER).equals("comma") ? "," : ";";
+ }
do {
params.update(SupportedDataLakeQueryParameters.QP_PAGE, String.valueOf(i));
@@ -159,7 +166,7 @@ public class DataLakeManagementV4 {
boolean isFirst = true;
for (int i1 = 0; i1 < dataResult.getHeaders().size(); i1++) {
if (!isFirst) {
- outputStream.write(toBytes(";"));
+ outputStream.write(toBytes(delimiter));
}
isFirst = false;
outputStream.write(toBytes(dataResult.getHeaders().get(i1)));
@@ -175,7 +182,7 @@ public class DataLakeManagementV4 {
for (int i1 = 0; i1 < row.size(); i1++) {
Object element = row.get(i1);
if (!isFirstInRow) {
- outputStream.write(toBytes(";"));
+ outputStream.write(toBytes(delimiter));
}
isFirstInRow = false;
if (i1 == 0) {
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/SupportedDataLakeQueryParameters.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/SupportedDataLakeQueryParameters.java
index 3a0862496..1eadd8124 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/SupportedDataLakeQueryParameters.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/SupportedDataLakeQueryParameters.java
@@ -33,6 +33,7 @@ public class SupportedDataLakeQueryParameters {
public static final String QP_AGGREGATION_FUNCTION = "aggregationFunction";
public static final String QP_TIME_INTERVAL = "timeInterval";
public static final String QP_FORMAT = "format";
+ public static final String QP_CSV_DELIMITER = "delimiter";
public static final String QP_COUNT_ONLY = "countOnly";
public static final String QP_AUTO_AGGREGATE = "autoAggregate";
public static final String QP_FILTER = "filter";
@@ -50,6 +51,7 @@ public class SupportedDataLakeQueryParameters {
QP_AGGREGATION_FUNCTION,
QP_TIME_INTERVAL,
QP_FORMAT,
+ QP_CSV_DELIMITER,
QP_COUNT_ONLY,
QP_AUTO_AGGREGATE,
QP_FILTER,
diff --git a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
index fcd1e6ea9..92397a221 100644
--- a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
+++ b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
@@ -184,6 +184,7 @@ public class DataLakeResourceV4 extends AbstractRestResource {
, @Parameter(in = ParameterIn.QUERY, description = "name of aggregation function used for grouping operation") @QueryParam(QP_AGGREGATION_FUNCTION) String aggregationFunction
, @Parameter(in = ParameterIn.QUERY, description = "time interval for aggregation (e.g. 1m - one minute) for grouping operation") @QueryParam(QP_TIME_INTERVAL) String timeInterval
, @Parameter(in = ParameterIn.QUERY, description = "format specification (csv, json - default is csv) for data download") @QueryParam(QP_FORMAT) String format
+ , @Parameter(in = ParameterIn.QUERY, description = "csv delimiter (comma or semicolon)") @QueryParam(QP_CSV_DELIMITER) String csvDelimiter
, @Parameter(in = ParameterIn.QUERY, description = "filter conditions (a comma-separated list of filter conditions such as [field,operator,condition])") @QueryParam(QP_FILTER) String filter
, @Context UriInfo uriInfo) {
diff --git a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
index 2b4aeb999..a4fedb77e 100644
--- a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
@@ -52,7 +52,7 @@ export class DatalakeRestService {
const columns = queryParams.columns;
if (columns === '') {
const emptyQueryResult = new SpQueryResult();
- emptyQueryResult.total = 0
+ emptyQueryResult.total = 0;
return of(emptyQueryResult);
} else {
const url = this.dataLakeUrl + '/measurements/' + index;
@@ -75,30 +75,36 @@ export class DatalakeRestService {
getTagValues(index: string,
fieldNames: string[]): Observable<Map<string, string[]>> {
-
+
if (fieldNames.length === 0) {
return of(new Map<string, string[]>());
} else {
return this.http.get(this.dataLakeUrl + '/measurements/' + index + '/tags?fields=' + fieldNames.toString())
- .pipe(map(r => r as Map<string, string[]>));
+ .pipe(map(r => r as Map<string, string[]>));
}
}
downloadRawData(index: string,
format: string,
+ delimiter: string,
startTime?: number,
endTime?: number) {
- const queryParams = (startTime && endTime) ? {format, startDate: startTime, endDate: endTime} : {format};
+ const queryParams = (startTime && endTime) ? {format, delimiter, startDate: startTime, endDate: endTime} : {
+ format,
+ delimiter
+ };
return this.buildDownloadRequest(index, queryParams);
}
downloadQueriedData(
index: string,
format: string,
+ delimiter: string,
queryParams: DatalakeQueryParameters) {
(queryParams as any).format = format;
+ (queryParams as any).delimiter = delimiter;
return this.buildDownloadRequest(index, queryParams);
}
diff --git a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.html b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.html
index 4eee9e73a..4c8675a26 100644
--- a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.html
+++ b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.html
@@ -23,22 +23,22 @@
<mat-step>
<ng-template matStepLabel>Select Data</ng-template>
<div class="mt-10">
- <mat-radio-group class="example-radio-group" [(ngModel)]="selectedData">
- <mat-radio-button value="visible" class="example-radio-button" *ngIf="dataConfig">
+ <mat-radio-group class="sp-radio-group" [(ngModel)]="selectedData">
+ <mat-radio-button value="visible" class="sp-radio-button" *ngIf="dataConfig">
Currently configured query
</mat-radio-button>
<div fxLayout="column" fxLayoutAlign="start start" class="ml-35 mb-10" *ngIf="selectedData === 'visible' && dataConfig && dataConfig.sourceConfigs.length > 1">
<h5>Select source (only one source can be exported in a single file)</h5>
- <mat-radio-group class="example-radio-group" [(ngModel)]="selectedQueryIndex" fxLayout="column">
+ <mat-radio-group class="sp-radio-group" [(ngModel)]="selectedQueryIndex" fxLayout="column">
<mat-radio-button [value]="i" *ngFor="let sourceConfig of dataConfig.sourceConfigs; let i = index" class="p-5">
{{sourceConfig.measureName}}
</mat-radio-button>
</mat-radio-group>
</div>
- <mat-radio-button value="all" class="example-radio-button">
+ <mat-radio-button value="all" class="sp-radio-button">
All data in database
</mat-radio-button>
- <mat-radio-button value="customInterval" class="example-radio-button">
+ <mat-radio-button value="customInterval" class="sp-radio-button">
All data in custom time interval
</mat-radio-button>
</mat-radio-group>
@@ -63,15 +63,23 @@
<mat-step>
<ng-template matStepLabel>Select Format</ng-template>
<div>
- <mat-radio-group class="example-radio-group" [(ngModel)]="downloadFormat">
- <mat-radio-button value="json" class="example-radio-button">
+ <h5>Download Format</h5>
+ <mat-radio-group class="sp-radio-group" [(ngModel)]="downloadFormat">
+ <mat-radio-button value="json" class="sp-radio-button">
JSON
</mat-radio-button>
- <mat-radio-button value="csv" class="example-radio-button">
+ <mat-radio-button value="csv" class="sp-radio-button">
CSV
</mat-radio-button>
</mat-radio-group>
</div>
+ <div *ngIf="downloadFormat === 'csv'" class="mt-10">
+ <h5>Delimiter</h5>
+ <mat-radio-group [(ngModel)]="delimiter" class="sp-radio-group">
+ <mat-radio-button value="comma" class="sp-radio-button"> ,</mat-radio-button>
+ <mat-radio-button value="semicolon" class="sp-radio-button"> ;</mat-radio-button>
+ </mat-radio-group>
+ </div>
</mat-step>
diff --git a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.scss b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.scss
index fd0f2f09b..1a799afd6 100644
--- a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.scss
+++ b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.scss
@@ -18,12 +18,12 @@
@import 'src/scss/sp/sp-dialog';
-example-radio-group {
+.sp-radio-group {
display: flex;
flex-direction: column;
margin: 15px 0;
}
-.example-radio-button {
+.sp-radio-button {
margin: 5px;
width: 100%;
}
diff --git a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
index b15054aab..092c87899 100644
--- a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
+++ b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
@@ -17,13 +17,14 @@
*/
import { HttpEventType } from '@angular/common/http';
-import { Component, Inject, Input, OnInit, ViewChild } from '@angular/core';
+import { Component, Input, OnInit, ViewChild } from '@angular/core';
import { MatStepper } from '@angular/material/stepper';
import {
- DataExplorerDataConfig, DatalakeQueryParameters,
+ DataExplorerDataConfig,
+ DatalakeQueryParameters,
DatalakeRestService,
DataViewQueryGeneratorService,
- DateRange, SourceConfig
+ DateRange
} from '@streampipes/platform-services';
import { DialogRef } from '@streampipes/shared-ui';
@@ -41,6 +42,7 @@ export class DataDownloadDialogComponent implements OnInit {
@Input() dataConfig: DataExplorerDataConfig;
downloadFormat = 'csv';
+ delimiter = ',';
selectedData = 'visible';
downloadFinish = false;
downloadedMBs: number = undefined;
@@ -80,11 +82,18 @@ export class DataDownloadDialogComponent implements OnInit {
const endDateString = this.getDateString(this.date.endDate);
switch (this.selectedData) {
case 'all':
- this.performRequest(this.datalakeRestService.downloadRawData(this.index, this.downloadFormat), '', '');
+ this.performRequest(this.datalakeRestService.downloadRawData(
+ this.index,
+ this.downloadFormat,
+ this.delimiter), '', '');
break;
case 'customInterval':
- this.performRequest(this.datalakeRestService.downloadRawData(this.index, this.downloadFormat,
- startTime, endTime), startDateString,
+ this.performRequest(this.datalakeRestService.downloadRawData(
+ this.index,
+ this.downloadFormat,
+ this.delimiter,
+ startTime,
+ endTime), startDateString,
endDateString);
break;
case 'visible':
@@ -93,6 +102,7 @@ export class DataDownloadDialogComponent implements OnInit {
.downloadQueriedData(
this.index,
this.downloadFormat,
+ this.delimiter,
this.generateQueryRequest(startTime, endTime)
),
startDateString,