You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2021/12/27 10:19:18 UTC
[incubator-streampipes] 02/02: [STREAMPIPES-483] Add test to validate connect preprocessing rules
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch STREAMPIPES-483
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 8bb2935235908c50e98e7b77e1c0248f819c242c
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Mon Dec 27 11:19:09 2021 +0100
[STREAMPIPES-483] Add test to validate connect preprocessing rules
---
.../fixtures/connect/schemaRules/expected.csv | 2 +
ui/cypress/fixtures/connect/schemaRules/input.csv | 2 +
ui/cypress/support/utils/AdapterUtils.ts | 16 +++-
ui/cypress/support/utils/DataLakeUtils.ts | 17 +++--
ui/cypress/support/utils/FileManagementUtils.ts | 20 ++---
ui/cypress/tests/adapter/schemaRules.smoke.ts | 88 +++++++++++++++-------
.../event-property-row.component.html | 9 ++-
.../schema-editor-header.component.html | 11 ++-
.../edit-data-type/edit-data-type.component.html | 8 +-
9 files changed, 117 insertions(+), 56 deletions(-)
diff --git a/ui/cypress/fixtures/connect/schemaRules/expected.csv b/ui/cypress/fixtures/connect/schemaRules/expected.csv
new file mode 100644
index 0000000..6f81327
--- /dev/null
+++ b/ui/cypress/fixtures/connect/schemaRules/expected.csv
@@ -0,0 +1,2 @@
+count;temperature
+122.0;11.0
diff --git a/ui/cypress/fixtures/connect/schemaRules/input.csv b/ui/cypress/fixtures/connect/schemaRules/input.csv
new file mode 100644
index 0000000..f92f0b8
--- /dev/null
+++ b/ui/cypress/fixtures/connect/schemaRules/input.csv
@@ -0,0 +1,2 @@
+count;density;temperature
+122.0;62.0;11
diff --git a/ui/cypress/support/utils/AdapterUtils.ts b/ui/cypress/support/utils/AdapterUtils.ts
index 7eaadff..9697ae3 100644
--- a/ui/cypress/support/utils/AdapterUtils.ts
+++ b/ui/cypress/support/utils/AdapterUtils.ts
@@ -123,7 +123,7 @@ export class AdapterUtils {
cy.get('button').contains('Next').parent().click();
}
- private static configureFormat(adapterConfiguration: GenericAdapterInput) {
+ public static configureFormat(adapterConfiguration: GenericAdapterInput) {
// Select format
cy.dataCy(adapterConfiguration.format).click();
@@ -142,9 +142,9 @@ export class AdapterUtils {
}
- private static markPropertyAsTimestamp(propertyName: string) {
+ public static markPropertyAsTimestamp(propertyName: string) {
// Mark property as timestamp
- cy.get('#event-schema-next-button').should('be.disabled');
+ this.eventSchemaNextBtnDisabled();
// Edit timestamp
cy.dataCy('edit-' + propertyName, { timeout: 10000 }).click();
@@ -154,6 +154,14 @@ export class AdapterUtils {
// Close
cy.dataCy('sp-save-edit-property').click();
+ this.eventSchemaNextBtnEnabled();
+ }
+
+ public static eventSchemaNextBtnDisabled() {
+ cy.get('#event-schema-next-button').should('be.disabled');
+ }
+
+ public static eventSchemaNextBtnEnabled() {
cy.get('#event-schema-next-button').parent().should('not.be.disabled');
}
@@ -167,7 +175,7 @@ export class AdapterUtils {
AdapterUtils.startAdapter(adapterInput, 'sp-connect-adapter-live-preview');
}
- private static startSetAdapter(adapterInput: AdapterInput) {
+ public static startSetAdapter(adapterInput: AdapterInput) {
AdapterUtils.startAdapter(adapterInput, 'sp-connect-adapter-set-success');
}
diff --git a/ui/cypress/support/utils/DataLakeUtils.ts b/ui/cypress/support/utils/DataLakeUtils.ts
index 35f7421..2dfd50a 100644
--- a/ui/cypress/support/utils/DataLakeUtils.ts
+++ b/ui/cypress/support/utils/DataLakeUtils.ts
@@ -162,7 +162,7 @@ export class DataLakeUtils {
cy.get('div').contains('DataLake').parent().click();
}
- public static checkResults(dataLakeIndex: string, fileRoute: string) {
+ public static checkResults(dataLakeIndex: string, fileRoute: string, ignoreTime: boolean = false) {
// Validate result in datalake
cy.request({
@@ -177,7 +177,7 @@ export class DataLakeUtils {
}).should((response) => {
const actualResultString = response.body;
cy.readFile(fileRoute).then((expectedResultString) => {
- DataLakeUtils.resultEqual(actualResultString, expectedResultString);
+ DataLakeUtils.resultEqual(actualResultString, expectedResultString, ignoreTime);
});
});
}
@@ -199,10 +199,17 @@ export class DataLakeUtils {
.type(`${value.toLocaleString()}{enter}`);
}
- private static resultEqual(expected: string, actual: string) {
+ private static resultEqual(actual: string, expected: string, ignoreTime: boolean) {
+ let actualResult;
+ if (ignoreTime) {
+ actualResult = DataLakeUtils.parseCsv(actual).map(row => {
+ return row.splice(1);
+ });
+ } else {
+ actualResult = DataLakeUtils.parseCsv(actual);
+ }
const expectedResult = DataLakeUtils.parseCsv(expected);
- const actualResult = DataLakeUtils.parseCsv(actual);
- expect(expectedResult).to.deep.equal(actualResult);
+ expect(actualResult).to.deep.equal(expectedResult);
}
private static parseCsv(csv: string) {
diff --git a/ui/cypress/support/utils/FileManagementUtils.ts b/ui/cypress/support/utils/FileManagementUtils.ts
index 13b0d01..581a02c 100644
--- a/ui/cypress/support/utils/FileManagementUtils.ts
+++ b/ui/cypress/support/utils/FileManagementUtils.ts
@@ -21,24 +21,24 @@ export class FileManagementUtils {
public static addFile(filePath: string) {
// Go to StreamPipes file management
- cy.visit('#/files');
+ cy.visit('#/files');
// Open file upload dialog
- cy.dataCy('sp-open-file-upload-dialog').click();
+ cy.dataCy('sp-open-file-upload-dialog').click();
// Upload file
- // const filepath = 'fileTest/test.csv'
- cy.dataCy('sp-file-management-file-input').attachFile(filePath);
- cy.dataCy('sp-file-management-store-file').click();
+ // const filepath = 'fileTest/test.csv'
+ cy.dataCy('sp-file-management-file-input').attachFile(filePath);
+ cy.dataCy('sp-file-management-store-file').click();
}
public static deleteFile() {
// Go to StreamPipes file management
- cy.visit('#/files');
+ cy.visit('#/files');
// Check if file was uploaded and delete it
- cy.dataCy('delete').should('have.length', 1);
- cy.dataCy('delete').click();
- cy.dataCy('confirm-delete').click();
- cy.dataCy('delete').should('have.length', 0);
+ cy.dataCy('delete').should('have.length', 1);
+ cy.dataCy('delete').click();
+ cy.dataCy('confirm-delete').click();
+ cy.dataCy('delete').should('have.length', 0);
}
}
diff --git a/ui/cypress/tests/adapter/schemaRules.smoke.ts b/ui/cypress/tests/adapter/schemaRules.smoke.ts
index a922e73..d0be54e 100644
--- a/ui/cypress/tests/adapter/schemaRules.smoke.ts
+++ b/ui/cypress/tests/adapter/schemaRules.smoke.ts
@@ -18,59 +18,89 @@
import { AdapterUtils } from '../../support/utils/AdapterUtils';
import { SpecificAdapterBuilder } from '../../support/builder/SpecificAdapterBuilder';
+import { FileManagementUtils } from '../../support/utils/FileManagementUtils';
+import { GenericAdapterBuilder } from '../../support/builder/GenericAdapterBuilder';
+import { DataLakeUtils } from '../../support/utils/DataLakeUtils';
describe('Test Random Data Simulator Stream Adapter', () => {
beforeEach('Setup Test', () => {
cy.initStreamPipesTest();
+ FileManagementUtils.addFile('connect/schemaRules/input.csv');
});
it('Perform Test', () => {
- const adapterConfiguration = SpecificAdapterBuilder
- .create('Machine_Data_Simulator')
- .setName('Machine Data Simulator Test')
- .addInput('input', 'wait-time-ms', '1000')
+ const adapterConfiguration = GenericAdapterBuilder
+ .create('File_Set')
+ .setStoreInDataLake()
+ .setTimestampProperty('timestamp')
+ .setName('Adapter to test schema rules')
+ .setFormat('csv')
+ .addFormatInput('input', 'delimiter', ';')
+ .addFormatInput('checkbox', 'header', 'check')
.build();
AdapterUtils.goToConnect();
-
AdapterUtils.selectAdapter(adapterConfiguration.adapterType);
-
- AdapterUtils.configureAdapter(adapterConfiguration.adapterConfiguration);
+ AdapterUtils.configureAdapter(adapterConfiguration.protocolConfiguration);
+ AdapterUtils.configureFormat(adapterConfiguration);
// wait till schema is shown
-
cy.dataCy('sp-connect-schema-editor', { timeout: 60000 }).should('be.visible');
- // Add fixed property
+ // Add static value to event
+ // TODO FIX breaks adapter
// Click add a static value to event
- cy.dataCy('connect-add-static-property', { timeout: 10000 }).click();
-
- // Edit new property
- const propertyName = 'staticPropertyName';
- const propertyValue = 'id1';
- cy.dataCy('edit-key_0', { timeout: 10000 }).click();
-
- cy.dataCy('connect-edit-field-runtime-name', { timeout: 10000 })
- .type('{backspace}{backspace}{backspace}{backspace}{backspace}' + propertyName);
- cy.dataCy('connect-edit-field-static-value', { timeout: 10000 }).type(propertyValue);
-
- cy.dataCy('sp-save-edit-property').click();
+ // cy.dataCy('connect-add-static-property', { timeout: 10000 }).click();
+ //
+ // // Edit new property
+ // const propertyName = 'staticPropertyName';
+ // const propertyValue = 'id1';
+ // cy.dataCy('edit-key_0', { timeout: 10000 }).click();
+ //
+ // cy.dataCy('connect-edit-field-runtime-name', { timeout: 10000 })
+ // .type('{backspace}{backspace}{backspace}{backspace}{backspace}' + propertyName);
+ // cy.dataCy('connect-edit-field-static-value', { timeout: 10000 }).type(propertyValue);
+ //
+ // cy.dataCy('sp-save-edit-property').click();
+ //
+ // // validate that static value is persisted
+ // cy.dataCy('edit-' + propertyName.toLowerCase(), { timeout: 10000 }).click();
+ // cy.dataCy('connect-edit-field-static-value', { timeout: 10000 }).should('have.value', propertyValue);
+ // cy.dataCy('sp-save-edit-property').click();
+
+ // Delete property
+ cy.dataCy('delete-property-density', { timeout: 10000 }).children().click({ force: true });
+ cy.dataCy('connect-schema-delete-properties-btn', { timeout: 10000 }).click();
+
+ // TODO FIX breaks adapter
+ // Change data type
+ // cy.dataCy('edit-temperature', { timeout: 10000 }).click();
+ // cy.dataCy('connect-change-runtime-type').click().get('mat-option').contains('Float').click();
+ // cy.dataCy('sp-save-edit-property').click();
+ // // validate that static value is persisted
+ // cy.dataCy('edit-temperature', { timeout: 10000 }).click({ force: true });
+ // cy.dataCy('connect-change-runtime-type', { timeout: 10000 }).contains('Float');
+ // cy.dataCy('sp-save-edit-property').click();
- // validate that static value is persisted
- cy.dataCy('edit-' + propertyName, { timeout: 10000 }).click();
- cy.dataCy('connect-edit-field-static-value', { timeout: 10000 }).should('have.value', propertyValue);
- cy.dataCy('sp-save-edit-property').click();
+ // Add timestamp
+ AdapterUtils.eventSchemaNextBtnDisabled();
+ cy.dataCy('connect-schema-add-timestamp-btn', { timeout: 10000 }).click();
+ AdapterUtils.eventSchemaNextBtnEnabled();
AdapterUtils.finishEventSchemaConfiguration();
- AdapterUtils.startAdapter(adapterConfiguration, 'sp-connect-adapter-live-preview');
+ AdapterUtils.startSetAdapter(adapterConfiguration);
- // Add timestamp
- // Delete
- // Change data type
+ // Wait till data is stored
+ cy.wait(10000);
+
+ DataLakeUtils.checkResults(
+ 'adaptertotestschemarules',
+ 'cypress/fixtures/connect/schemaRules/expected.csv',
+ true);
});
});
diff --git a/ui/src/app/connect/components/schema-editor/event-property-row/event-property-row.component.html b/ui/src/app/connect/components/schema-editor/event-property-row/event-property-row.component.html
index 8fda374..df17659 100644
--- a/ui/src/app/connect/components/schema-editor/event-property-row/event-property-row.component.html
+++ b/ui/src/app/connect/components/schema-editor/event-property-row/event-property-row.component.html
@@ -47,15 +47,18 @@
*ngIf="isNested || isPrimitive || isList">
<button [disabled]="!isEditable" color="accent" mat-button
(click)="openEditDialog(node.data)"
- [attr.data-cy]="'edit-' + label">
+ [attr.data-cy]="'edit-' + label.toLowerCase()">
<mat-icon>edit</mat-icon>
</button>
</div>
<div fxFlex="0 1 auto" fxLayoutAlign="center center">
<mat-checkbox
*ngIf="isNested || isPrimitive || isList"
- (click)="selectProperty(node.data.id, undefined)" [disabled]="!isEditable"
- [class.checkbox-selected]="node.data.selected" [checked]="node.data.selected">
+ (click)="selectProperty(node.data.id, undefined)"
+ [disabled]="!isEditable"
+ [class.checkbox-selected]="node.data.selected"
+ [checked]="node.data.selected"
+ [attr.data-cy]="'delete-property-' + label.toLowerCase()">
</mat-checkbox>
</div>
</div>
diff --git a/ui/src/app/connect/components/schema-editor/schema-editor-header/schema-editor-header.component.html b/ui/src/app/connect/components/schema-editor/schema-editor-header/schema-editor-header.component.html
index b3cae9a..2c30634 100644
--- a/ui/src/app/connect/components/schema-editor/schema-editor-header/schema-editor-header.component.html
+++ b/ui/src/app/connect/components/schema-editor/schema-editor-header/schema-editor-header.component.html
@@ -34,7 +34,8 @@
<button color="primary"
mat-button
data-cy="connect-add-timestamp-property"
- (click)=addTimestampProperty()>
+ (click)=addTimestampProperty()
+ data-cy="connect-schema-add-timestamp-btn">
<mat-icon matTooltip="Add timestamp to event schema">access_time</mat-icon>
</button>
<button color="primary"
@@ -53,8 +54,12 @@
</sp-error-hint>
- <button style="padding-right:0" color="primary" mat-button [disabled]="countSelected == 0"
- (click)="removeSelectedProperties()">
+ <button style="padding-right:0"
+ color="primary"
+ mat-button
+ [disabled]="countSelected == 0"
+ (click)="removeSelectedProperties()"
+ data-cy="connect-schema-delete-properties-btn">
<mat-icon matTooltip="Remove selected Properties">delete</mat-icon>
</button>
</div>
diff --git a/ui/src/app/connect/dialog/edit-event-property/components/edit-data-type/edit-data-type.component.html b/ui/src/app/connect/dialog/edit-event-property/components/edit-data-type/edit-data-type.component.html
index f8da0c0..7e8c9f4 100644
--- a/ui/src/app/connect/dialog/edit-event-property/components/edit-data-type/edit-data-type.component.html
+++ b/ui/src/app/connect/dialog/edit-event-property/components/edit-data-type/edit-data-type.component.html
@@ -17,8 +17,12 @@
-->
<mat-form-field class="doubleWidth" color="accent">
- <mat-select placeholder="Runtime Type" [(ngModel)]="cachedProperty.runtimeType" (selectionChange)="valueChanged()">
- <mat-option *ngFor="let dataType of runtimeDataTypes" [value]="dataType.url">
+ <mat-select placeholder="Runtime Type"
+ [(ngModel)]="cachedProperty.runtimeType"
+ (selectionChange)="valueChanged()"
+ data-cy="connect-change-runtime-type">
+ <mat-option *ngFor="let dataType of runtimeDataTypes"
+ [value]="dataType.url">
{{ dataType.label }}
</mat-option>
</mat-select>