You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2022/08/11 19:54:06 UTC
[incubator-streampipes] 03/04: [STREAMPIPES-573] Add delimiter selection to data download dialog
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 16f97b68a3b423660a173f88f98fc4ba4dc8d7ec
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Thu Aug 11 21:53:43 2022 +0200
[STREAMPIPES-573] Add delimiter selection to data download dialog
---
.../dataexplorer/DataLakeManagementV4.java | 15 ++++++++++----
.../v4/SupportedDataLakeQueryParameters.java | 2 ++
.../apache/streampipes/ps/DataLakeResourceV4.java | 1 +
.../src/lib/apis/datalake-rest.service.ts | 14 +++++++++----
.../data-download-dialog.component.html | 24 ++++++++++++++--------
.../data-download-dialog.component.scss | 4 ++--
.../data-download-dialog.component.ts | 22 ++++++++++++++------
7 files changed, 58 insertions(+), 24 deletions(-)
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java
index 523cbe8aa..8f6a858ff 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java
@@ -36,7 +36,10 @@ import org.apache.streampipes.model.datalake.DataLakeConfiguration;
import org.apache.streampipes.model.datalake.DataLakeMeasure;
import org.apache.streampipes.model.datalake.DataLakeRetentionPolicy;
import org.apache.streampipes.model.datalake.SpQueryResult;
-import org.apache.streampipes.model.schema.*;
+import org.apache.streampipes.model.schema.EventProperty;
+import org.apache.streampipes.model.schema.EventPropertyList;
+import org.apache.streampipes.model.schema.EventPropertyNested;
+import org.apache.streampipes.model.schema.EventPropertyPrimitive;
import org.apache.streampipes.storage.api.IDataLakeStorage;
import org.apache.streampipes.storage.couchdb.utils.Utils;
import org.apache.streampipes.storage.management.StorageDispatcher;
@@ -45,7 +48,6 @@ import org.influxdb.dto.Query;
import org.influxdb.dto.QueryResult;
import org.lightcouch.CouchDbClient;
-import javax.xml.crypto.Data;
import java.io.IOException;
import java.io.OutputStream;
import java.time.Instant;
@@ -149,6 +151,11 @@ public class DataLakeManagementV4 {
}
boolean isFirstDataObject = true;
+ String delimiter = ",";
+
+ if (params.has(QP_CSV_DELIMITER)) {
+ delimiter = params.getAsString(QP_CSV_DELIMITER).equals("comma") ? "," : ";";
+ }
do {
params.update(SupportedDataLakeQueryParameters.QP_PAGE, String.valueOf(i));
@@ -159,7 +166,7 @@ public class DataLakeManagementV4 {
boolean isFirst = true;
for (int i1 = 0; i1 < dataResult.getHeaders().size(); i1++) {
if (!isFirst) {
- outputStream.write(toBytes(";"));
+ outputStream.write(toBytes(delimiter));
}
isFirst = false;
outputStream.write(toBytes(dataResult.getHeaders().get(i1)));
@@ -175,7 +182,7 @@ public class DataLakeManagementV4 {
for (int i1 = 0; i1 < row.size(); i1++) {
Object element = row.get(i1);
if (!isFirstInRow) {
- outputStream.write(toBytes(";"));
+ outputStream.write(toBytes(delimiter));
}
isFirstInRow = false;
if (i1 == 0) {
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/SupportedDataLakeQueryParameters.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/SupportedDataLakeQueryParameters.java
index 3a0862496..1eadd8124 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/SupportedDataLakeQueryParameters.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/v4/SupportedDataLakeQueryParameters.java
@@ -33,6 +33,7 @@ public class SupportedDataLakeQueryParameters {
public static final String QP_AGGREGATION_FUNCTION = "aggregationFunction";
public static final String QP_TIME_INTERVAL = "timeInterval";
public static final String QP_FORMAT = "format";
+ public static final String QP_CSV_DELIMITER = "delimiter";
public static final String QP_COUNT_ONLY = "countOnly";
public static final String QP_AUTO_AGGREGATE = "autoAggregate";
public static final String QP_FILTER = "filter";
@@ -50,6 +51,7 @@ public class SupportedDataLakeQueryParameters {
QP_AGGREGATION_FUNCTION,
QP_TIME_INTERVAL,
QP_FORMAT,
+ QP_CSV_DELIMITER,
QP_COUNT_ONLY,
QP_AUTO_AGGREGATE,
QP_FILTER,
diff --git a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
index fcd1e6ea9..92397a221 100644
--- a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
+++ b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
@@ -184,6 +184,7 @@ public class DataLakeResourceV4 extends AbstractRestResource {
, @Parameter(in = ParameterIn.QUERY, description = "name of aggregation function used for grouping operation") @QueryParam(QP_AGGREGATION_FUNCTION) String aggregationFunction
, @Parameter(in = ParameterIn.QUERY, description = "time interval for aggregation (e.g. 1m - one minute) for grouping operation") @QueryParam(QP_TIME_INTERVAL) String timeInterval
, @Parameter(in = ParameterIn.QUERY, description = "format specification (csv, json - default is csv) for data download") @QueryParam(QP_FORMAT) String format
+ , @Parameter(in = ParameterIn.QUERY, description = "csv delimiter (comma or semicolon)") @QueryParam(QP_CSV_DELIMITER) String csvDelimiter
, @Parameter(in = ParameterIn.QUERY, description = "filter conditions (a comma-separated list of filter conditions such as [field,operator,condition])") @QueryParam(QP_FILTER) String filter
, @Context UriInfo uriInfo) {
diff --git a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
index 2b4aeb999..a4fedb77e 100644
--- a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
@@ -52,7 +52,7 @@ export class DatalakeRestService {
const columns = queryParams.columns;
if (columns === '') {
const emptyQueryResult = new SpQueryResult();
- emptyQueryResult.total = 0
+ emptyQueryResult.total = 0;
return of(emptyQueryResult);
} else {
const url = this.dataLakeUrl + '/measurements/' + index;
@@ -75,30 +75,36 @@ export class DatalakeRestService {
getTagValues(index: string,
fieldNames: string[]): Observable<Map<string, string[]>> {
-
+
if (fieldNames.length === 0) {
return of(new Map<string, string[]>());
} else {
return this.http.get(this.dataLakeUrl + '/measurements/' + index + '/tags?fields=' + fieldNames.toString())
- .pipe(map(r => r as Map<string, string[]>));
+ .pipe(map(r => r as Map<string, string[]>));
}
}
downloadRawData(index: string,
format: string,
+ delimiter: string,
startTime?: number,
endTime?: number) {
- const queryParams = (startTime && endTime) ? {format, startDate: startTime, endDate: endTime} : {format};
+ const queryParams = (startTime && endTime) ? {format, delimiter, startDate: startTime, endDate: endTime} : {
+ format,
+ delimiter
+ };
return this.buildDownloadRequest(index, queryParams);
}
downloadQueriedData(
index: string,
format: string,
+ delimiter: string,
queryParams: DatalakeQueryParameters) {
(queryParams as any).format = format;
+ (queryParams as any).delimiter = delimiter;
return this.buildDownloadRequest(index, queryParams);
}
diff --git a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.html b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.html
index 4eee9e73a..4c8675a26 100644
--- a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.html
+++ b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.html
@@ -23,22 +23,22 @@
<mat-step>
<ng-template matStepLabel>Select Data</ng-template>
<div class="mt-10">
- <mat-radio-group class="example-radio-group" [(ngModel)]="selectedData">
- <mat-radio-button value="visible" class="example-radio-button" *ngIf="dataConfig">
+ <mat-radio-group class="sp-radio-group" [(ngModel)]="selectedData">
+ <mat-radio-button value="visible" class="sp-radio-button" *ngIf="dataConfig">
Currently configured query
</mat-radio-button>
<div fxLayout="column" fxLayoutAlign="start start" class="ml-35 mb-10" *ngIf="selectedData === 'visible' && dataConfig && dataConfig.sourceConfigs.length > 1">
<h5>Select source (only one source can be exported in a single file)</h5>
- <mat-radio-group class="example-radio-group" [(ngModel)]="selectedQueryIndex" fxLayout="column">
+ <mat-radio-group class="sp-radio-group" [(ngModel)]="selectedQueryIndex" fxLayout="column">
<mat-radio-button [value]="i" *ngFor="let sourceConfig of dataConfig.sourceConfigs; let i = index" class="p-5">
{{sourceConfig.measureName}}
</mat-radio-button>
</mat-radio-group>
</div>
- <mat-radio-button value="all" class="example-radio-button">
+ <mat-radio-button value="all" class="sp-radio-button">
All data in database
</mat-radio-button>
- <mat-radio-button value="customInterval" class="example-radio-button">
+ <mat-radio-button value="customInterval" class="sp-radio-button">
All data in custom time interval
</mat-radio-button>
</mat-radio-group>
@@ -63,15 +63,23 @@
<mat-step>
<ng-template matStepLabel>Select Format</ng-template>
<div>
- <mat-radio-group class="example-radio-group" [(ngModel)]="downloadFormat">
- <mat-radio-button value="json" class="example-radio-button">
+ <h5>Download Format</h5>
+ <mat-radio-group class="sp-radio-group" [(ngModel)]="downloadFormat">
+ <mat-radio-button value="json" class="sp-radio-button">
JSON
</mat-radio-button>
- <mat-radio-button value="csv" class="example-radio-button">
+ <mat-radio-button value="csv" class="sp-radio-button">
CSV
</mat-radio-button>
</mat-radio-group>
</div>
+ <div *ngIf="downloadFormat === 'csv'" class="mt-10">
+ <h5>Delimiter</h5>
+ <mat-radio-group [(ngModel)]="delimiter" class="sp-radio-group">
+ <mat-radio-button value="comma" class="sp-radio-button"> ,</mat-radio-button>
+ <mat-radio-button value="semicolon" class="sp-radio-button"> ;</mat-radio-button>
+ </mat-radio-group>
+ </div>
</mat-step>
diff --git a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.scss b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.scss
index fd0f2f09b..1a799afd6 100644
--- a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.scss
+++ b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.scss
@@ -18,12 +18,12 @@
@import 'src/scss/sp/sp-dialog';
-example-radio-group {
+.sp-radio-group {
display: flex;
flex-direction: column;
margin: 15px 0;
}
-.example-radio-button {
+.sp-radio-button {
margin: 5px;
width: 100%;
}
diff --git a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
index b15054aab..092c87899 100644
--- a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
+++ b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
@@ -17,13 +17,14 @@
*/
import { HttpEventType } from '@angular/common/http';
-import { Component, Inject, Input, OnInit, ViewChild } from '@angular/core';
+import { Component, Input, OnInit, ViewChild } from '@angular/core';
import { MatStepper } from '@angular/material/stepper';
import {
- DataExplorerDataConfig, DatalakeQueryParameters,
+ DataExplorerDataConfig,
+ DatalakeQueryParameters,
DatalakeRestService,
DataViewQueryGeneratorService,
- DateRange, SourceConfig
+ DateRange
} from '@streampipes/platform-services';
import { DialogRef } from '@streampipes/shared-ui';
@@ -41,6 +42,7 @@ export class DataDownloadDialogComponent implements OnInit {
@Input() dataConfig: DataExplorerDataConfig;
downloadFormat = 'csv';
+ delimiter = ',';
selectedData = 'visible';
downloadFinish = false;
downloadedMBs: number = undefined;
@@ -80,11 +82,18 @@ export class DataDownloadDialogComponent implements OnInit {
const endDateString = this.getDateString(this.date.endDate);
switch (this.selectedData) {
case 'all':
- this.performRequest(this.datalakeRestService.downloadRawData(this.index, this.downloadFormat), '', '');
+ this.performRequest(this.datalakeRestService.downloadRawData(
+ this.index,
+ this.downloadFormat,
+ this.delimiter), '', '');
break;
case 'customInterval':
- this.performRequest(this.datalakeRestService.downloadRawData(this.index, this.downloadFormat,
- startTime, endTime), startDateString,
+ this.performRequest(this.datalakeRestService.downloadRawData(
+ this.index,
+ this.downloadFormat,
+ this.delimiter,
+ startTime,
+ endTime), startDateString,
endDateString);
break;
case 'visible':
@@ -93,6 +102,7 @@ export class DataDownloadDialogComponent implements OnInit {
.downloadQueriedData(
this.index,
this.downloadFormat,
+ this.delimiter,
this.generateQueryRequest(startTime, endTime)
),
startDateString,