You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2022/10/01 19:44:49 UTC
[incubator-streampipes] branch STREAMPIPES-583 updated: [STREAMPIPES-583] Updated file naming for the download
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch STREAMPIPES-583
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
The following commit(s) were added to refs/heads/STREAMPIPES-583 by this push:
new 7d433066c [STREAMPIPES-583] Updated file naming for the download
7d433066c is described below
commit 7d433066ca14493aac1df651580607595a0ae8f8
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Sat Oct 1 21:44:25 2022 +0200
[STREAMPIPES-583] Updated file naming for the download
---
.../data-download-dialog.component.ts | 1 -
.../services/data-export.service.spec.ts | 98 ++++++++++++
.../services/data-export.service.ts | 168 ++++-----------------
ui/tsconfig.json | 2 +-
4 files changed, 132 insertions(+), 137 deletions(-)
diff --git a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
index c9aa11dd1..f388ce2f1 100644
--- a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
+++ b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
@@ -71,7 +71,6 @@ export class DataDownloadDialogComponent implements OnInit {
}
downloadData() {
- // TODO check if I still need it
if (this.exportConfig.dataExportConfig.dataRangeConfiguration === 'visible') {
this.exportConfig.dataExportConfig.dateRange = this.dataDownloadDialogModel.dataExplorerDateRange;
}
diff --git a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.spec.ts b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.spec.ts
new file mode 100644
index 000000000..f99c5d1c8
--- /dev/null
+++ b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.spec.ts
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { DataExportService } from './data-export.service';
+import { ExportConfig } from '../model/export-config.model';
+import { DateRange } from '@streampipes/platform-services';
+
+describe('DataExportService', () => {
+
+ const service = new DataExportService(undefined, undefined);
+
+ // Testing data
+ const defaultExportDate = new Date(2022, 11, 11, 3);
+ const defaultStartDate = new Date(2022, 11, 2, 3);
+ const defaultEndDate = new Date(2022, 11, 6, 3);
+ const defaultDateRange = new DateRange(defaultStartDate, defaultEndDate);
+ let defaultExportConfig: ExportConfig;
+
+ const expectedBaseName = '2022-12-11_measurement_';
+
+ beforeEach(() => {
+ defaultExportConfig = {
+ dataExportConfig: {
+ dataRangeConfiguration: 'all',
+ missingValueBehaviour: 'ignore',
+ measurement: 'measurement'
+ },
+ formatExportConfig: {
+ exportFormat: 'csv',
+ delimiter: 'comma'
+ }
+ };
+ });
+
+ it('Name for all data csv', () => {
+ const result = service.getFileName(defaultExportConfig, defaultExportDate);
+ expect(result).toBe(`${expectedBaseName}all.csv`);
+ });
+
+ it('Name for all data json', () => {
+ defaultExportConfig.formatExportConfig.exportFormat = 'json';
+ const result = service.getFileName(defaultExportConfig, defaultExportDate);
+ expect(result).toBe(`${expectedBaseName}all.json`);
+ });
+
+ it('Name for custom interval csv', () => {
+ defaultExportConfig.dataExportConfig.dataRangeConfiguration = 'customInterval';
+ defaultExportConfig.dataExportConfig.dateRange = defaultDateRange;
+
+ const result = service.getFileName(defaultExportConfig, defaultExportDate);
+ expect(result).toBe(`${expectedBaseName}customInterval_2022-12-02_2022-12-06.csv`);
+ });
+
+ it('Name for custom visible json', () => {
+ defaultExportConfig.formatExportConfig.exportFormat = 'json';
+ defaultExportConfig.dataExportConfig.dataRangeConfiguration = 'visible';
+ defaultExportConfig.dataExportConfig.dateRange = defaultDateRange;
+
+ const result = service.getFileName(defaultExportConfig, defaultExportDate);
+ expect(result).toBe(`${expectedBaseName}visible_2022-12-02_2022-12-06.json`);
+ });
+
+ it('Name when missing start date range', () => {
+ defaultExportConfig.dataExportConfig.dataRangeConfiguration = 'visible';
+ defaultExportConfig.dataExportConfig.dateRange = new DateRange();
+ defaultExportConfig.dataExportConfig.dateRange.endDate = defaultEndDate;
+
+ const result = service.getFileName(defaultExportConfig, defaultExportDate);
+ expect(result).toBe(`${expectedBaseName}visible_2022-12-06.csv`);
+ });
+
+ it('Name when missing end date range', () => {
+ defaultExportConfig.dataExportConfig.dataRangeConfiguration = 'visible';
+ defaultExportConfig.dataExportConfig.dateRange = new DateRange();
+ defaultExportConfig.dataExportConfig.dateRange.startDate = defaultStartDate;
+
+ const result = service.getFileName(defaultExportConfig, defaultExportDate);
+ expect(result).toBe(`${expectedBaseName}visible_2022-12-02.csv`);
+ });
+
+})
+;
+
diff --git a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
index fa58fef70..d6ecddebe 100644
--- a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
+++ b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
@@ -35,7 +35,7 @@ export class DataExportService {
}
public downloadData(exportConfig: ExportConfig,
- dataDownloadDialogModel: DataDownloadDialogModel) {
+ dataDownloadDialogModel: DataDownloadDialogModel) {
let downloadRequest;
@@ -43,14 +43,14 @@ export class DataExportService {
downloadRequest = this.dataLakeRestService.downloadQueriedData(
exportConfig.dataExportConfig.measurement,
exportConfig.formatExportConfig.exportFormat,
- exportConfig.formatExportConfig.exportFormat['delimiter'],
+ exportConfig.formatExportConfig['delimiter'],
this.generateQueryRequest(exportConfig, dataDownloadDialogModel));
} else {
// case for 'all' & 'customInterval'
downloadRequest = this.dataLakeRestService.downloadRawData(
exportConfig.dataExportConfig.measurement,
exportConfig.formatExportConfig.exportFormat,
- exportConfig.formatExportConfig.exportFormat['delimiter']);
+ exportConfig.formatExportConfig['delimiter']);
}
downloadRequest.subscribe(event => {
@@ -66,7 +66,7 @@ export class DataExportService {
// finished
if (event.type === HttpEventType.Response) {
- // this.createFile(event.body, this.downloadFormat, this.measureName, startDate, endDate);
+ this.createFile(event.body, exportConfig);
downloadProgress = {
downloadedMBs: event.loaded / 1024 / 1014,
finished: true,
@@ -74,59 +74,6 @@ export class DataExportService {
this.updateDownloadProgress.emit(downloadProgress);
}
});
-
-
- // const index = !this.dataExplorerDataConfig ?
- // this.measureName : this.dataExplorerDataConfig.sourceConfigs[this.selectedQueryIndex].measureName;
- // // this.nextStep();
- // const startTime = this.date.startDate.getTime();
- // const endTime = this.date.endDate.getTime();
- // const startDateString = this.getDateString(this.date.startDate);
- // const endDateString = this.getDateString(this.date.endDate);
-
- // this.performRequest(
- // this.datalakeRestService.downloadRawData(
- // exportConfig.dataExportConfig.measurement,
- // exportConfig.formatExportConfig.exportFormat,
- // exportConfig.formatExportConfig.exportFormat['delimiter']),
- // '',
- // '');
- // this.performRequest(
- // this.datalakeRestService
- // .downloadQueriedData(
- // index,
- // this.downloadFormat,
- // this.delimiter,
- // this.generateQueryRequest(startTime, endTime)
- // ),
- // startDateString,
- // endDateString
- // );
- // this.performRequest(
- // this.datalakeRestService.downloadRawData(
- // exportConfig.dataExportConfig.measurement,
- // exportConfig.formatExportConfig.exportFormat,
- // exportConfig.formatExportConfig.exportFormat['delimiter']),
- // exportConfig.dataExportConfig.dateRange.startDate,
- // exportConfig.dataExportConfig.dateRange.endDate),
- // this.getDateString(exportConfig.dataExportConfig.dateRange.startDate),
- // this.getDateString(exportConfig.dataExportConfig.dateRange.endDate);
- // )
- // ;
- // break;
- // case 'visible':
- // this.performRequest(
- // this.datalakeRestService
- // .downloadQueriedData(
- // index,
- // this.downloadFormat,
- // this.delimiter,
- // this.generateQueryRequest(startTime, endTime)
- // ),
- // startDateString,
- // endDateString
- // );
-
}
// TODO how to set the selected Query Index
@@ -141,99 +88,50 @@ export class DataExportService {
dataDownloadDialogModel.dataExplorerDataConfig.sourceConfigs[selectedQueryIndex]);
}
- // performRequest(request, startDate, endDate) {
- // const downloadHttpRequestSubscription = request.subscribe(event => {
- // let downloadProgress: DownloadProgress;
- // // progress
- // if (event.type === HttpEventType.DownloadProgress) {
- // downloadProgress = {
- // downloadedMBs: event.loaded / 1024 / 1014,
- // finished: false,
- // };
- // // this.downloadedMBs = event.loaded / 1024 / 1014;
- // }
- //
- // // finished
- // if (event.type === HttpEventType.Response) {
- // // this.createFile(event.body, this.downloadFormat, this.measureName, startDate, endDate);
- // downloadProgress = {
- // finished: true,
- // };
- //
- // // this.downloadFinish = true;
- // }
- // });
- // }
-
-
- private createFile(data, format, fileName, startDate, endDate, downloadFormat) {
+ /**
+ * The code in this method can be updated. Currently, it uses a native approach to download a file.
+ * At other places we use the fileSaver library (https://www.npmjs.com/package/file-saver) to download files.
+ * However, this library only supports files up to 2GB.
+ * There are other alternatives like StreamSaver.js (https://github.com/jimmywarting/StreamSaver.js).
+ * But then we probably should replace the fileSaver library with StreamSaver to only have one dependency.
+ * In the documentation it is also stated that an alternative approach is under development.
+ * It is probably also worth in looking into this one.
+ */
+ private createFile(data, exportConfig: ExportConfig) {
const a = document.createElement('a');
document.body.appendChild(a);
a.style.display = 'display: none';
- let name = 'sp_' + startDate + '_' + fileName + '.' + downloadFormat;
- name = name.replace('__', '_');
+ const name = this.getFileName(exportConfig, new Date());
- const url = window.URL.createObjectURL(new Blob([String(data)], {type: 'data:text/' + format + ';charset=utf-8'}));
+ const url = window.URL.createObjectURL(new Blob([String(data)], {type: `data:text/${exportConfig.formatExportConfig.exportFormat};charset=utf-8`}));
a.href = url;
a.download = name;
a.click();
window.URL.revokeObjectURL(url);
}
- private getDateString(date: Date): string {
- return date.toLocaleDateString() + 'T' + date.toLocaleTimeString().replace(':', '.')
- .replace(':', '.');
- }
-
- private convertData(data, format, xAxesKey, yAxesKeys) {
- const indexXKey = data.headers.findIndex(headerName => headerName === xAxesKey);
- const indicesYKeys = [];
- yAxesKeys.forEach(key => {
- indicesYKeys.push(data.headers.findIndex(headerName => headerName === key));
- });
+ public getFileName(exportConfig: ExportConfig, exportDate: Date): string {
+ const baseName = `${this.getDateString(exportDate)}_${exportConfig.dataExportConfig.measurement}_`;
+ const dataRangeOption = exportConfig.dataExportConfig.dataRangeConfiguration;
+ let dateRange = '';
+ const fileExtension = `.${exportConfig.formatExportConfig.exportFormat}`;
- if (format === 'json') {
- const resultJson = [];
+ if (exportConfig.dataExportConfig.dateRange !== undefined && (dataRangeOption === 'customInterval' || dataRangeOption === 'visible')) {
+ if (exportConfig.dataExportConfig.dateRange.startDate) {
+ dateRange += `_${this.getDateString(exportConfig.dataExportConfig.dateRange.startDate)}`;
+ }
+ if (exportConfig.dataExportConfig.dateRange.endDate) {
+ dateRange += `_${this.getDateString(exportConfig.dataExportConfig.dateRange.endDate)}`;
+ }
+ }
- data.rows.forEach(row => {
- const tmp = {'time': new Date(row[indexXKey]).getTime()};
- indicesYKeys.forEach(index => {
- if (row[index] !== undefined) {
- tmp[data.headers[index]] = row[index];
- }
- });
- resultJson.push(tmp);
- });
+ return baseName + dataRangeOption + dateRange + fileExtension;
+ }
- return JSON.stringify(resultJson);
- } else {
- // CSV
- let resultCsv = '';
-
- // header
- resultCsv += xAxesKey;
- yAxesKeys.forEach(key => {
- resultCsv += ';';
- resultCsv += key;
- });
-
-
- // content
- data.rows.forEach(row => {
- resultCsv += '\n';
- resultCsv += new Date(row[indexXKey]).getTime();
- indicesYKeys.forEach(index => {
- resultCsv += ';';
- if (row[index] !== undefined) {
- resultCsv += row[index];
- }
- });
- });
-
- return resultCsv;
- }
+ public getDateString(date: Date): string {
+ return date.toISOString().split('T')[0];
}
}
diff --git a/ui/tsconfig.json b/ui/tsconfig.json
index adf3a71f2..725c0180b 100644
--- a/ui/tsconfig.json
+++ b/ui/tsconfig.json
@@ -26,7 +26,7 @@
"node"
],
"lib": [
- "es2017",
+ "es2021",
"dom"
],
"baseUrl": "./"