You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2021/08/30 13:48:13 UTC
[incubator-streampipes] 01/02: [STREAMPIPES-421] Configure
collection static properties CSV file
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit bad9271eda4de04d838b4f0bcdf74e50e38b9242
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Mon Aug 30 15:17:22 2021 +0200
[STREAMPIPES-421] Configure collection static properties CSV file
---
ui/cypress/support/utils/DataLakeUtils.ts | 44 +++----
.../add-to-collection.component.html | 9 +-
.../add-to-collection.component.spec.ts | 39 +++++++
.../add-to-collection.component.ts | 129 ++++++++++++---------
4 files changed, 142 insertions(+), 79 deletions(-)
diff --git a/ui/cypress/support/utils/DataLakeUtils.ts b/ui/cypress/support/utils/DataLakeUtils.ts
index be763cf..b1b1438 100644
--- a/ui/cypress/support/utils/DataLakeUtils.ts
+++ b/ui/cypress/support/utils/DataLakeUtils.ts
@@ -22,30 +22,30 @@ import { UserUtils } from './UserUtils';
export class DataLakeUtils {
- public static checkResults(dataLakeIndex: string, fileRoute: string) {
+ public static checkResults(dataLakeIndex: string, fileRoute: string) {
- // Validate result in datalake
- cy.request('GET', '/streampipes-backend/api/v3/users/' + UserUtils.testUserName + '/datalake/data/' + dataLakeIndex + '/download?format=csv',
- {'content-type': 'application/octet-stream'}).should((response) => {
- const expectedResultString = response.body;
- cy.readFile(fileRoute).then((actualResultString) => {
- DataLakeUtils.resultEqual(actualResultString, expectedResultString);
- });
- });
- }
+ // Validate result in datalake
+ cy.request('GET', '/streampipes-backend/api/v3/users/' + UserUtils.testUserName + '/datalake/data/' + dataLakeIndex + '/download?format=csv',
+ { 'content-type': 'application/octet-stream' }).should((response) => {
+ const expectedResultString = response.body;
+ cy.readFile(fileRoute).then((actualResultString) => {
+ DataLakeUtils.resultEqual(actualResultString, expectedResultString);
+ });
+ });
+ }
- private static resultEqual(actual: string, expected: string) {
- const expectedResult = DataLakeUtils.parseCsv(expected);
- const actualResult = DataLakeUtils.parseCsv(actual);
- expect(expectedResult).to.deep.equal(actualResult);
- }
+ private static resultEqual(actual: string, expected: string) {
+ const expectedResult = DataLakeUtils.parseCsv(expected);
+ const actualResult = DataLakeUtils.parseCsv(actual);
+ expect(expectedResult).to.deep.equal(actualResult);
+ }
- private static parseCsv(csv: string) {
- const result = [];
- const index = CSV.readAll(csv, row => {
- result.push(row);
- });
+ private static parseCsv(csv: string) {
+ const result = [];
+ const index = CSV.readAll(csv, row => {
+ result.push(row);
+ });
- return result;
- }
+ return result;
+ }
}
diff --git a/ui/src/app/core-ui/static-properties/static-collection/add-to-collection/add-to-collection.component.html b/ui/src/app/core-ui/static-properties/static-collection/add-to-collection/add-to-collection.component.html
index 7fb5297..381bf3f 100644
--- a/ui/src/app/core-ui/static-properties/static-collection/add-to-collection/add-to-collection.component.html
+++ b/ui/src/app/core-ui/static-properties/static-collection/add-to-collection/add-to-collection.component.html
@@ -31,7 +31,7 @@
<span>Back</span>
</button>
<div fxFlex="100" style="margin:5px;width:100%">
- <mat-form-field style="width: 95%" (click)="fileInput.click();" color="accent">
+ <mat-form-field style="width: 95%" (click)="fileInput.click();" color="accent" *ngIf="!hasError">
<input matInput placeholder="File" disabled>
<input #fileInput type="file" style="display:none;"
data-cy="sp-file-management-file-input"
@@ -44,9 +44,10 @@
mat-button style="min-width: 0px">
<mat-icon>insert_drive_file</mat-icon>
</button>
- <mat-error>
- {{errorMessage}}
- </mat-error>
</mat-form-field>
+ <mat-error *ngIf="hasError">
+ {{errorMessage}}
+ </mat-error>
</div>
+
</div>
\ No newline at end of file
diff --git a/ui/src/app/core-ui/static-properties/static-collection/add-to-collection/add-to-collection.component.spec.ts b/ui/src/app/core-ui/static-properties/static-collection/add-to-collection/add-to-collection.component.spec.ts
new file mode 100644
index 0000000..46dac5c
--- /dev/null
+++ b/ui/src/app/core-ui/static-properties/static-collection/add-to-collection/add-to-collection.component.spec.ts
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+import { AddToCollectionComponent } from './add-to-collection.component';
+
+describe('AddToCollectionComponent', () => {
+ const component: AddToCollectionComponent = new AddToCollectionComponent(undefined);
+
+ it('parse csv string', () => {
+ const csvString = [
+ 'a,b',
+ 'a1,b1',
+ 'a2,b2'
+ ].join('\n');
+
+ const result = component.parseCsv(csvString);
+
+ result.subscribe(res => {
+ expect(res.length).toBe(2);
+ expect(res[0]).toEqual({ 'a': 'a1', 'b': 'b1' });
+ expect(res[1]).toEqual({ 'a': 'a2', 'b': 'b2' });
+ });
+ });
+});
+
diff --git a/ui/src/app/core-ui/static-properties/static-collection/add-to-collection/add-to-collection.component.ts b/ui/src/app/core-ui/static-properties/static-collection/add-to-collection/add-to-collection.component.ts
index 1fcdb1e..79d76f9 100644
--- a/ui/src/app/core-ui/static-properties/static-collection/add-to-collection/add-to-collection.component.ts
+++ b/ui/src/app/core-ui/static-properties/static-collection/add-to-collection/add-to-collection.component.ts
@@ -18,7 +18,8 @@
import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
import { StaticPropertyUtilService } from '../../static-property-util.service';
-import { EventProperty, OneOfStaticProperty, StaticProperty } from '../../../../core-model/gen/streampipes-model';
+import { FreeTextStaticProperty, OneOfStaticProperty, StaticProperty } from '../../../../core-model/gen/streampipes-model';
+import { Observable } from 'rxjs';
@Component({
selector: 'sp-add-to-collection',
@@ -31,7 +32,7 @@ export class AddToCollectionComponent implements OnInit {
public staticPropertyTemplate: StaticProperty;
@Output()
- addPropertyEmitter: EventEmitter<EventProperty> = new EventEmitter<EventProperty>();
+ addPropertyEmitter: EventEmitter<StaticProperty> = new EventEmitter<StaticProperty>();
public showFileSelecion = false;
@@ -40,7 +41,7 @@ export class AddToCollectionComponent implements OnInit {
public fileName: string;
public hasError = false;
- public errorMessage = '';
+ public errorMessage = 'This is a test';
constructor(private staticPropertyUtil: StaticPropertyUtilService) {
}
@@ -61,55 +62,39 @@ export class AddToCollectionComponent implements OnInit {
this.showFileSelecion = false;
this.processingFile = false;
this.hasError = false;
+ this.fileName = '';
+ this.errorMessage = '';
}
handleFileInput(target: any) {
+
this.processingFile = true;
const fileReader = new FileReader();
this.fileName = target.files[0].name;
fileReader.onload = (e) => {
- const res = this.parseCsv(fileReader.result);
- res.pop();
-
- // TODO Error: Wrong headers in csv file
-
- let finalProperties: EventProperty[] = [];
-
- res.forEach((row, i) => {
- const clone = this.staticPropertyUtil.clone(this.staticPropertyTemplate);
-
- // Check that all values are within csv row
- clone.staticProperties.forEach(p => {
- if (p instanceof OneOfStaticProperty) {
- const option = p.options.find(o => o.name === row[p.label]);
- if (!option) {
- option.selected = true;
- } else {
- this.setError('Error in line ' + i + '. Value for ' + p.label + ' is not supported');
- }
- } else {
- if (row[p.label] === '') {
- this.setError('Error in line ' + i + '. Value for ' + p.label + ' is not set');
- } else {
- p.value = row[p.label];
- }
- }
- finalProperties.push(clone);
+ this.parseCsv(fileReader.result).subscribe(res => {
+ res.pop();
+ res.forEach((row, i) => {
+ const property: StaticProperty = this.getStaticProperty(row, i);
+ finalProperties.push(property);
});
- });
-
- if (!this.hasError) {
- finalProperties.forEach(p => {
- this.addPropertyEmitter.emit(p);
- });
- this.closeFileSelection();
- this.fileName = '';
- }
+ if (!this.hasError) {
+ finalProperties.forEach(p => {
+ this.addPropertyEmitter.emit(p);
+ });
+ this.closeFileSelection();
+ this.fileName = '';
+ }
+ });
};
+
fileReader.readAsText(target.files[0]);
+
+ // Parse file and return properties
+ const finalProperties: StaticProperty[] = [];
}
private setError(errorMessage: string) {
@@ -119,22 +104,60 @@ export class AddToCollectionComponent implements OnInit {
}
}
- private parseCsv(str) {
+ public parseCsv(str): Observable<any[]> {
+
str = str.replace(/\r?\n|\r/g, '\n');
- const delimiter = ';';
- const headers = str.slice(0, str.indexOf('\n')).split(delimiter);
-
- const rows = str.slice(str.indexOf('\n') + 1).split('\n');
-
- const arr = rows.map(row => {
- const values = row.split(delimiter);
- const el = headers.reduce((object, header, index) => {
- object[header] = values[index];
- return object;
- }, {});
- return el;
+ const parseResult = new Observable<any[]>((observer) => {
+ const delimiter = ',';
+ const headers = str.slice(0, str.indexOf('\n')).split(delimiter);
+
+ const rows = str.slice(str.indexOf('\n') + 1).split('\n');
+
+ const result = rows.map(row => {
+ const values = row.split(delimiter);
+ const el = headers.reduce((object, header, index) => {
+ object[header] = values[index];
+ return object;
+ }, {});
+ return el;
+ });
+
+ observer.next(result);
+
});
- return arr;
+ return parseResult;
+ }
+
+ public getStaticProperty(row: any, rowNumber: number): StaticProperty {
+ const clone = this.staticPropertyUtil.clone(this.staticPropertyTemplate);
+
+ // Check that all values are within csv row
+ clone.staticProperties.forEach(p => {
+ if (p instanceof OneOfStaticProperty) {
+ this.setOneOfStaticProperty(row, p, rowNumber);
+ } else {
+ this.setStaticProperty(row, p, rowNumber);
+ }
+ });
+
+ return clone;
+ }
+
+ private setOneOfStaticProperty(row: any, property: OneOfStaticProperty, rowNumber: number) {
+ const option = property.options.find(o => o.name === row[property.label]);
+ if (option !== undefined) {
+ option.selected = true;
+ } else {
+ this.setError('Error in line ' + rowNumber + '. Value for "' + property.label + '" is not supported');
+ }
+ }
+
+ private setStaticProperty(row: any, property: FreeTextStaticProperty, rowNumber: number) {
+ if (row[property.label] === undefined || row[property.label] === '') {
+ this.setError('Error in line ' + rowNumber + '. Value for "' + property.label + '" is not set');
+ } else {
+ property.value = row[property.label];
+ }
}
}