You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2022/10/01 19:44:49 UTC

[incubator-streampipes] branch STREAMPIPES-583 updated: [STREAMPIPES-583] Updated file naming for the download

This is an automated email from the ASF dual-hosted git repository.

zehnder pushed a commit to branch STREAMPIPES-583
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git


The following commit(s) were added to refs/heads/STREAMPIPES-583 by this push:
     new 7d433066c [STREAMPIPES-583] Updated file naming for the download
7d433066c is described below

commit 7d433066ca14493aac1df651580607595a0ae8f8
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Sat Oct 1 21:44:25 2022 +0200

    [STREAMPIPES-583] Updated file naming for the download
---
 .../data-download-dialog.component.ts              |   1 -
 .../services/data-export.service.spec.ts           |  98 ++++++++++++
 .../services/data-export.service.ts                | 168 ++++-----------------
 ui/tsconfig.json                                   |   2 +-
 4 files changed, 132 insertions(+), 137 deletions(-)

diff --git a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
index c9aa11dd1..f388ce2f1 100644
--- a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
+++ b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
@@ -71,7 +71,6 @@ export class DataDownloadDialogComponent implements OnInit {
   }
 
   downloadData() {
-    // TODO check if I still need it
     if (this.exportConfig.dataExportConfig.dataRangeConfiguration === 'visible') {
      this.exportConfig.dataExportConfig.dateRange = this.dataDownloadDialogModel.dataExplorerDateRange;
     }
diff --git a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.spec.ts b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.spec.ts
new file mode 100644
index 000000000..f99c5d1c8
--- /dev/null
+++ b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.spec.ts
@@ -0,0 +1,98 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+import { DataExportService } from './data-export.service';
+import { ExportConfig } from '../model/export-config.model';
+import { DateRange } from '@streampipes/platform-services';
+
+describe('DataExportService', () => {
+
+  const service = new DataExportService(undefined, undefined);
+
+  // Testing data
+  const defaultExportDate = new Date(2022, 11, 11, 3);
+  const defaultStartDate = new Date(2022, 11, 2, 3);
+  const defaultEndDate = new Date(2022, 11, 6, 3);
+  const defaultDateRange = new DateRange(defaultStartDate, defaultEndDate);
+  let defaultExportConfig: ExportConfig;
+
+  const expectedBaseName = '2022-12-11_measurement_';
+
+  beforeEach(() => {
+    defaultExportConfig = {
+      dataExportConfig: {
+        dataRangeConfiguration: 'all',
+        missingValueBehaviour: 'ignore',
+        measurement: 'measurement'
+      },
+      formatExportConfig: {
+        exportFormat: 'csv',
+        delimiter: 'comma'
+      }
+    };
+  });
+
+  it('Name for all data csv', () => {
+    const result = service.getFileName(defaultExportConfig, defaultExportDate);
+    expect(result).toBe(`${expectedBaseName}all.csv`);
+  });
+
+  it('Name for all data json', () => {
+    defaultExportConfig.formatExportConfig.exportFormat = 'json';
+    const result = service.getFileName(defaultExportConfig, defaultExportDate);
+    expect(result).toBe(`${expectedBaseName}all.json`);
+  });
+
+  it('Name for custom interval csv', () => {
+    defaultExportConfig.dataExportConfig.dataRangeConfiguration = 'customInterval';
+    defaultExportConfig.dataExportConfig.dateRange = defaultDateRange;
+
+    const result = service.getFileName(defaultExportConfig, defaultExportDate);
+    expect(result).toBe(`${expectedBaseName}customInterval_2022-12-02_2022-12-06.csv`);
+  });
+
+  it('Name for custom visible json', () => {
+    defaultExportConfig.formatExportConfig.exportFormat = 'json';
+    defaultExportConfig.dataExportConfig.dataRangeConfiguration = 'visible';
+    defaultExportConfig.dataExportConfig.dateRange = defaultDateRange;
+
+    const result = service.getFileName(defaultExportConfig, defaultExportDate);
+    expect(result).toBe(`${expectedBaseName}visible_2022-12-02_2022-12-06.json`);
+  });
+
+  it('Name when missing start date range', () => {
+    defaultExportConfig.dataExportConfig.dataRangeConfiguration = 'visible';
+    defaultExportConfig.dataExportConfig.dateRange = new DateRange();
+    defaultExportConfig.dataExportConfig.dateRange.endDate = defaultEndDate;
+
+    const result = service.getFileName(defaultExportConfig, defaultExportDate);
+    expect(result).toBe(`${expectedBaseName}visible_2022-12-06.csv`);
+  });
+
+  it('Name when missing end date range', () => {
+    defaultExportConfig.dataExportConfig.dataRangeConfiguration = 'visible';
+    defaultExportConfig.dataExportConfig.dateRange = new DateRange();
+    defaultExportConfig.dataExportConfig.dateRange.startDate = defaultStartDate;
+
+    const result = service.getFileName(defaultExportConfig, defaultExportDate);
+    expect(result).toBe(`${expectedBaseName}visible_2022-12-02.csv`);
+  });
+
+})
+;
+
diff --git a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
index fa58fef70..d6ecddebe 100644
--- a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
+++ b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
@@ -35,7 +35,7 @@ export class DataExportService {
   }
 
   public downloadData(exportConfig: ExportConfig,
-               dataDownloadDialogModel: DataDownloadDialogModel) {
+                      dataDownloadDialogModel: DataDownloadDialogModel) {
 
     let downloadRequest;
 
@@ -43,14 +43,14 @@ export class DataExportService {
       downloadRequest = this.dataLakeRestService.downloadQueriedData(
         exportConfig.dataExportConfig.measurement,
         exportConfig.formatExportConfig.exportFormat,
-        exportConfig.formatExportConfig.exportFormat['delimiter'],
+        exportConfig.formatExportConfig['delimiter'],
         this.generateQueryRequest(exportConfig, dataDownloadDialogModel));
     } else {
       // case for 'all' & 'customInterval'
       downloadRequest = this.dataLakeRestService.downloadRawData(
         exportConfig.dataExportConfig.measurement,
         exportConfig.formatExportConfig.exportFormat,
-        exportConfig.formatExportConfig.exportFormat['delimiter']);
+        exportConfig.formatExportConfig['delimiter']);
     }
 
     downloadRequest.subscribe(event => {
@@ -66,7 +66,7 @@ export class DataExportService {
 
       // finished
       if (event.type === HttpEventType.Response) {
-        // this.createFile(event.body, this.downloadFormat, this.measureName, startDate, endDate);
+        this.createFile(event.body, exportConfig);
         downloadProgress = {
           downloadedMBs: event.loaded / 1024 / 1014,
           finished: true,
@@ -74,59 +74,6 @@ export class DataExportService {
         this.updateDownloadProgress.emit(downloadProgress);
       }
     });
-
-
-    // const index = !this.dataExplorerDataConfig ?
-    //   this.measureName : this.dataExplorerDataConfig.sourceConfigs[this.selectedQueryIndex].measureName;
-    // // this.nextStep();
-    // const startTime = this.date.startDate.getTime();
-    // const endTime = this.date.endDate.getTime();
-    // const startDateString = this.getDateString(this.date.startDate);
-    // const endDateString = this.getDateString(this.date.endDate);
-
-    // this.performRequest(
-    //   this.datalakeRestService.downloadRawData(
-    //     exportConfig.dataExportConfig.measurement,
-    //     exportConfig.formatExportConfig.exportFormat,
-    //     exportConfig.formatExportConfig.exportFormat['delimiter']),
-    //   '',
-    //   '');
-    // this.performRequest(
-    //   this.datalakeRestService
-    //     .downloadQueriedData(
-    //       index,
-    //       this.downloadFormat,
-    //       this.delimiter,
-    //       this.generateQueryRequest(startTime, endTime)
-    //     ),
-    //   startDateString,
-    //   endDateString
-    // );
-    //   this.performRequest(
-    //     this.datalakeRestService.downloadRawData(
-    //       exportConfig.dataExportConfig.measurement,
-    //       exportConfig.formatExportConfig.exportFormat,
-    //       exportConfig.formatExportConfig.exportFormat['delimiter']),
-    //     exportConfig.dataExportConfig.dateRange.startDate,
-    //     exportConfig.dataExportConfig.dateRange.endDate),
-    //     this.getDateString(exportConfig.dataExportConfig.dateRange.startDate),
-    //     this.getDateString(exportConfig.dataExportConfig.dateRange.endDate);
-    // )
-    //   ;
-    //   break;
-    // case 'visible':
-    //   this.performRequest(
-    //     this.datalakeRestService
-    //       .downloadQueriedData(
-    //         index,
-    //         this.downloadFormat,
-    //         this.delimiter,
-    //         this.generateQueryRequest(startTime, endTime)
-    //       ),
-    //     startDateString,
-    //     endDateString
-    //   );
-
   }
 
   // TODO how to set the selected Query Index
@@ -141,99 +88,50 @@ export class DataExportService {
         dataDownloadDialogModel.dataExplorerDataConfig.sourceConfigs[selectedQueryIndex]);
   }
 
-  // performRequest(request, startDate, endDate) {
-  //   const downloadHttpRequestSubscription = request.subscribe(event => {
-  //     let downloadProgress: DownloadProgress;
-  //     // progress
-  //     if (event.type === HttpEventType.DownloadProgress) {
-  //       downloadProgress = {
-  //         downloadedMBs: event.loaded / 1024 / 1014,
-  //         finished: false,
-  //       };
-  //       // this.downloadedMBs = event.loaded / 1024 / 1014;
-  //     }
-  //
-  //     // finished
-  //     if (event.type === HttpEventType.Response) {
-  //       // this.createFile(event.body, this.downloadFormat, this.measureName, startDate, endDate);
-  //       downloadProgress = {
-  //         finished: true,
-  //       };
-  //
-  //       // this.downloadFinish = true;
-  //     }
-  //   });
-  // }
-
-
-  private createFile(data, format, fileName, startDate, endDate, downloadFormat) {
+  /**
+   * The code in this method can be updated. Currently, it uses a native approach to download a file.
+   * At other places we use the fileSaver library (https://www.npmjs.com/package/file-saver) to download files.
+   * However, this library only supports files up to 2GB.
+   * There are other alternatives like StreamSaver.js (https://github.com/jimmywarting/StreamSaver.js).
+   * But then we probably should replace the fileSaver library with StreamSaver to only have one dependency.
+   * In the documentation it is also stated that an alternative approach is under development.
+   * It is probably also worth in looking into this one.
+   */
+  private createFile(data, exportConfig: ExportConfig) {
     const a = document.createElement('a');
     document.body.appendChild(a);
     a.style.display = 'display: none';
 
-    let name = 'sp_' + startDate + '_' + fileName + '.' + downloadFormat;
-    name = name.replace('__', '_');
+    const name = this.getFileName(exportConfig, new Date());
 
-    const url = window.URL.createObjectURL(new Blob([String(data)], {type: 'data:text/' + format + ';charset=utf-8'}));
+    const url = window.URL.createObjectURL(new Blob([String(data)], {type: `data:text/${exportConfig.formatExportConfig.exportFormat};charset=utf-8`}));
     a.href = url;
     a.download = name;
     a.click();
     window.URL.revokeObjectURL(url);
   }
 
-  private getDateString(date: Date): string {
-    return date.toLocaleDateString() + 'T' + date.toLocaleTimeString().replace(':', '.')
-      .replace(':', '.');
-  }
-
-  private convertData(data, format, xAxesKey, yAxesKeys) {
-    const indexXKey = data.headers.findIndex(headerName => headerName === xAxesKey);
-    const indicesYKeys = [];
-    yAxesKeys.forEach(key => {
-      indicesYKeys.push(data.headers.findIndex(headerName => headerName === key));
-    });
+  public getFileName(exportConfig: ExportConfig, exportDate: Date): string {
+    const baseName = `${this.getDateString(exportDate)}_${exportConfig.dataExportConfig.measurement}_`;
+    const dataRangeOption = exportConfig.dataExportConfig.dataRangeConfiguration;
+    let dateRange = '';
+    const fileExtension = `.${exportConfig.formatExportConfig.exportFormat}`;
 
-    if (format === 'json') {
-      const resultJson = [];
+    if (exportConfig.dataExportConfig.dateRange !== undefined && (dataRangeOption === 'customInterval' || dataRangeOption === 'visible')) {
+      if (exportConfig.dataExportConfig.dateRange.startDate) {
+        dateRange += `_${this.getDateString(exportConfig.dataExportConfig.dateRange.startDate)}`;
+      }
 
+      if (exportConfig.dataExportConfig.dateRange.endDate) {
+        dateRange += `_${this.getDateString(exportConfig.dataExportConfig.dateRange.endDate)}`;
+      }
+    }
 
-      data.rows.forEach(row => {
-        const tmp = {'time': new Date(row[indexXKey]).getTime()};
-        indicesYKeys.forEach(index => {
-          if (row[index] !== undefined) {
-            tmp[data.headers[index]] = row[index];
-          }
-        });
-        resultJson.push(tmp);
-      });
+    return baseName + dataRangeOption + dateRange + fileExtension;
+  }
 
-      return JSON.stringify(resultJson);
-    } else {
-      // CSV
-      let resultCsv = '';
-
-      // header
-      resultCsv += xAxesKey;
-      yAxesKeys.forEach(key => {
-        resultCsv += ';';
-        resultCsv += key;
-      });
-
-
-      // content
-      data.rows.forEach(row => {
-        resultCsv += '\n';
-        resultCsv += new Date(row[indexXKey]).getTime();
-        indicesYKeys.forEach(index => {
-          resultCsv += ';';
-          if (row[index] !== undefined) {
-            resultCsv += row[index];
-          }
-        });
-      });
-
-      return resultCsv;
-    }
+  public getDateString(date: Date): string {
+    return date.toISOString().split('T')[0];
   }
 
 }
diff --git a/ui/tsconfig.json b/ui/tsconfig.json
index adf3a71f2..725c0180b 100644
--- a/ui/tsconfig.json
+++ b/ui/tsconfig.json
@@ -26,7 +26,7 @@
       "node"
     ],
     "lib": [
-      "es2017",
+      "es2021",
       "dom"
     ],
     "baseUrl": "./"