You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2021/12/27 14:34:51 UTC
[incubator-streampipes] branch STREAMPIPES-483 updated: [STREAMPIPES-483] Add tests for value transformation rules
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch STREAMPIPES-483
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
The following commit(s) were added to refs/heads/STREAMPIPES-483 by this push:
new 864171c [STREAMPIPES-483] Add tests for value transformation rules
864171c is described below
commit 864171c8876296eeb84d5462f7ab6911ea54992b
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Mon Dec 27 15:34:40 2021 +0100
[STREAMPIPES-483] Add tests for value transformation rules
---
.../downloads/sp_adaptertotestschemarules.csv | 2 +
.../fixtures/connect/valueRules/expected.csv | 2 +
ui/cypress/fixtures/connect/valueRules/input.csv | 2 +
ui/cypress/tests/adapter/schemaRules.smoke.ts | 6 +-
ui/cypress/tests/adapter/valueRules.smoke.ts | 109 +++++++++++++++++++++
.../edit-correction-value.component.html | 10 +-
.../edit-timestamp-property.component.html | 7 +-
.../edit-unit-transformation.component.html | 39 ++++++--
8 files changed, 162 insertions(+), 15 deletions(-)
diff --git a/ui/cypress/downloads/sp_adaptertotestschemarules.csv b/ui/cypress/downloads/sp_adaptertotestschemarules.csv
new file mode 100644
index 0000000..c9ff8ef
--- /dev/null
+++ b/ui/cypress/downloads/sp_adaptertotestschemarules.csv
@@ -0,0 +1,2 @@
+time;value
+0;62.0
diff --git a/ui/cypress/fixtures/connect/valueRules/expected.csv b/ui/cypress/fixtures/connect/valueRules/expected.csv
new file mode 100644
index 0000000..8dbbb4e
--- /dev/null
+++ b/ui/cypress/fixtures/connect/valueRules/expected.csv
@@ -0,0 +1,2 @@
+time;temperature;value
+1640346912123;50.003334045410156;100.0
diff --git a/ui/cypress/fixtures/connect/valueRules/input.csv b/ui/cypress/fixtures/connect/valueRules/input.csv
new file mode 100644
index 0000000..072815a
--- /dev/null
+++ b/ui/cypress/fixtures/connect/valueRules/input.csv
@@ -0,0 +1,2 @@
+timestamp;value;temperature
+2021-12-24T12:55:12.123Z;10.0;10.0
diff --git a/ui/cypress/tests/adapter/schemaRules.smoke.ts b/ui/cypress/tests/adapter/schemaRules.smoke.ts
index 1e2192c..81d3c9b 100644
--- a/ui/cypress/tests/adapter/schemaRules.smoke.ts
+++ b/ui/cypress/tests/adapter/schemaRules.smoke.ts
@@ -22,7 +22,7 @@ import { GenericAdapterBuilder } from '../../support/builder/GenericAdapterBuild
import { ConnectEventSchemaUtils } from '../../support/utils/ConnectEventSchemaUtils';
import { DataLakeUtils } from '../../support/utils/DataLakeUtils';
-describe('Test Random Data Simulator Stream Adapter', () => {
+describe('Connect schema rule transformations', () => {
beforeEach('Setup Test', () => {
cy.initStreamPipesTest();
FileManagementUtils.addFile('connect/schemaRules/input.csv');
@@ -55,8 +55,8 @@ describe('Test Random Data Simulator Stream Adapter', () => {
// Delete one property
ConnectEventSchemaUtils.deleteProperty('density');
- // TODO use data type class
- ConnectEventSchemaUtils.changePropertyDataType('temperature', 'Float');
+ // Set data type to float
+ ConnectEventSchemaUtils.changePropertyDataType('temperature', 'Integer');
// Add a timestamp property
ConnectEventSchemaUtils.addTimestampProperty();
diff --git a/ui/cypress/tests/adapter/valueRules.smoke.ts b/ui/cypress/tests/adapter/valueRules.smoke.ts
new file mode 100644
index 0000000..8aea013
--- /dev/null
+++ b/ui/cypress/tests/adapter/valueRules.smoke.ts
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { ConnectUtils } from '../../support/utils/ConnectUtils';
+import { FileManagementUtils } from '../../support/utils/FileManagementUtils';
+import { GenericAdapterBuilder } from '../../support/builder/GenericAdapterBuilder';
+import { ConnectEventSchemaUtils } from '../../support/utils/ConnectEventSchemaUtils';
+import { DataLakeUtils } from '../../support/utils/DataLakeUtils';
+
+describe('Connect schema rule transformations', () => {
+ beforeEach('Setup Test', () => {
+ cy.initStreamPipesTest();
+ FileManagementUtils.addFile('connect/valueRules/input.csv');
+ });
+
+ it('Perform Test', () => {
+
+ const adapterConfiguration = GenericAdapterBuilder
+ .create('File_Set')
+ .setStoreInDataLake()
+ .setTimestampProperty('timestamp')
+ .setName('Adapter to test schema rules')
+ .setFormat('csv')
+ .addFormatInput('input', 'delimiter', ';')
+ .addFormatInput('checkbox', 'header', 'check')
+ .build();
+
+
+ ConnectUtils.goToConnect();
+ ConnectUtils.selectAdapter(adapterConfiguration.adapterType);
+ ConnectUtils.configureAdapter(adapterConfiguration.protocolConfiguration);
+ ConnectUtils.configureFormat(adapterConfiguration);
+
+ // wait till schema is shown
+ cy.dataCy('sp-connect-schema-editor', { timeout: 60000 }).should('be.visible');
+
+ // Edit timestamp property
+ let propertyName = 'timestamp';
+ const timestampRegex = 'yyyy-MM-dd\'T\'HH:mm:ss.SSS\'Z\'';
+ cy.dataCy('edit-' + propertyName.toLowerCase(), { timeout: 10000 }).click();
+ cy.dataCy('sp-mark-as-timestamp').children().click();
+ cy.dataCy('connect-timestamp-converter').click().get('mat-option').contains('String').click();
+ cy.dataCy('connect-timestamp-string-regex').type(timestampRegex);
+
+ cy.dataCy('sp-save-edit-property').click();
+
+ cy.dataCy('edit-' + propertyName.toLowerCase(), { timeout: 10000 }).click({ force: true });
+ cy.dataCy('connect-timestamp-string-regex', { timeout: 10000 }).should('have.value', timestampRegex);
+ cy.dataCy('sp-save-edit-property').click();
+
+
+ // Number transformation
+ propertyName = 'value';
+ const value = '10';
+ cy.dataCy('edit-' + propertyName.toLowerCase(), { timeout: 10000 }).click();
+ cy.dataCy('connect-schema-correction-value').type(value);
+ cy.dataCy('connect-schema-correction-operator').click().get('mat-option').contains('Multiply').click();
+
+ cy.dataCy('sp-save-edit-property').click();
+ cy.dataCy('edit-' + propertyName.toLowerCase(), { timeout: 10000 }).click({ force: true });
+ cy.dataCy('connect-schema-correction-value', { timeout: 10000 }).should('have.value', value);
+ cy.dataCy('sp-save-edit-property').click();
+
+ // Unit transformation
+ propertyName = 'temperature';
+ const fromUnit = '';
+ const toUnit = '';
+ cy.dataCy('edit-' + propertyName.toLowerCase(), { timeout: 10000 }).click();
+ cy.dataCy('connect-schema-unit-from-dropdown').type('Degree Celsius');
+ cy.dataCy('connect-schema-unit-transform-btn').click();
+ cy.dataCy('connect-schema-unit-to-dropdown').click().get('mat-option').contains('Degree Fahrenheit').click();
+ cy.dataCy('sp-save-edit-property').click();
+
+ // TODO fix check when editing
+ // cy.dataCy('edit-' + propertyName.toLowerCase(), { timeout: 10000 }).click({ force: true });
+ // cy.dataCy('connect-schema-unit-from-input', { timeout: 10000 }).should('have.value', 'Degree Celsius');
+ // cy.dataCy('connect-schema-unit-to-dropdown', { timeout: 10000 }).should('have.value', 'Degree Fahrenheit');
+ // cy.dataCy('sp-save-edit-property').click();
+
+
+ ConnectEventSchemaUtils.finishEventSchemaConfiguration();
+
+ ConnectUtils.startSetAdapter(adapterConfiguration);
+
+ // Wait till data is stored
+ cy.wait(10000);
+
+ DataLakeUtils.checkResults(
+ 'adaptertotestschemarules',
+ 'cypress/fixtures/connect/valueRules/expected.csv',
+ false);
+ });
+
+});
diff --git a/ui/src/app/connect/dialog/edit-event-property/components/edit-correction-value/edit-correction-value.component.html b/ui/src/app/connect/dialog/edit-event-property/components/edit-correction-value/edit-correction-value.component.html
index 718797b..049ebd8 100644
--- a/ui/src/app/connect/dialog/edit-event-property/components/edit-correction-value/edit-correction-value.component.html
+++ b/ui/src/app/connect/dialog/edit-event-property/components/edit-correction-value/edit-correction-value.component.html
@@ -19,7 +19,12 @@
<div fxLayout="row" fxLayoutAlign="start">
<div class="form-group" fxFlexAlign="center">
<mat-form-field class="example-full-width" style="margin-right: 10px" color="accent">
- <input matInput name="correction" placeholder="Correction Value" [(ngModel)]="cachedProperty.correctionValue" (change)="valueChanged()">
+ <input matInput
+ name="correction"
+ placeholder="Correction Value"
+ [(ngModel)]="cachedProperty.correctionValue"
+ (change)="valueChanged()"
+ data-cy="connect-schema-correction-value">
</mat-form-field>
</div>
<div class="form-group" fxFlexAlign="center">
@@ -29,7 +34,8 @@
placeholder="Math Operator"
[(ngModel)]="cachedProperty.operator"
(selectionChange)="valueChanged()"
- [disabled]="!cachedProperty.correctionValue">
+ [disabled]="!cachedProperty.correctionValue"
+ data-cy="connect-schema-correction-operator">
<mat-option [value]="op.value" *ngFor="let op of operators">
{{ op.viewValue }}
</mat-option>
diff --git a/ui/src/app/connect/dialog/edit-event-property/components/edit-timestamp-property/edit-timestamp-property.component.html b/ui/src/app/connect/dialog/edit-event-property/components/edit-timestamp-property/edit-timestamp-property.component.html
index 54f046e..111b137 100644
--- a/ui/src/app/connect/dialog/edit-event-property/components/edit-timestamp-property/edit-timestamp-property.component.html
+++ b/ui/src/app/connect/dialog/edit-event-property/components/edit-timestamp-property/edit-timestamp-property.component.html
@@ -19,7 +19,8 @@
<div fxLayout="row" fxLayoutAlign="start">
<mat-form-field class="doubleWidth" color="accent">
<mat-select placeholder="Timestamp converter (unix timestamp)"
- [(value)]="cachedProperty.timestampTransformationMode">
+ [(value)]="cachedProperty.timestampTransformationMode"
+ data-cy="connect-timestamp-converter">
<mat-option value="none">None</mat-option>
<mat-option value="formatString">Date String</mat-option>
<mat-option value="timeUnit">Date Number</mat-option>
@@ -31,7 +32,9 @@
<div fxLayout="row" fxLayoutAlign="start">
<div class="form-group" fxFlexAlign="center">
<mat-form-field class="doubleWidth" *ngIf="cachedProperty.timestampTransformationMode === 'formatString'" color="accent">
- <input matInput placeholder="E.g. yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
+ <input matInput
+ placeholder="E.g. yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
+ data-cy="connect-timestamp-string-regex"
[(ngModel)]="cachedProperty.timestampTransformationFormatString">
</mat-form-field>
<mat-form-field class="doubleWidth" *ngIf="cachedProperty.timestampTransformationMode === 'timeUnit'" color="accent">
diff --git a/ui/src/app/connect/dialog/edit-event-property/components/edit-unit-transformation/edit-unit-transformation.component.html b/ui/src/app/connect/dialog/edit-event-property/components/edit-unit-transformation/edit-unit-transformation.component.html
index 43a8b05..edc51e2 100644
--- a/ui/src/app/connect/dialog/edit-event-property/components/edit-unit-transformation/edit-unit-transformation.component.html
+++ b/ui/src/app/connect/dialog/edit-event-property/components/edit-unit-transformation/edit-unit-transformation.component.html
@@ -19,26 +19,49 @@
<div fxLayout="row" fxLayoutAlign="start" *ngIf="showUnitTransformation">
<div class="form-group" fxFlexAlign="center" style="width: 155px">
<mat-form-field *ngIf="!hadMeasurementUnit" class="example-full-width" style="width: 155px" color="accent">
- <input matInput placeholder="Unit" [matAutocomplete]="auto" [formControl]="stateCtrl"
- [attr.disabled]="transformUnitEnable ? '' : null">
+ <input matInput
+ placeholder="Unit"
+ [matAutocomplete]="auto"
+ [formControl]="stateCtrl"
+ [attr.disabled]="transformUnitEnable ? '' : null"
+ data-cy="connect-schema-unit-from-dropdown">
<mat-autocomplete #auto="matAutocomplete" [displayWith]="displayFn" [panelWidth]="'300px'">
- <mat-option *ngFor="let unit of filteredUnits | async" [value]="unit.label">
+ <mat-option
+ *ngFor="let unit of filteredUnits | async"
+ [value]="unit.label"
+ [attr.data-cy]="unit.resource">
{{unit.label}}
</mat-option>
</mat-autocomplete>
</mat-form-field>
- <mat-form-field *ngIf="hadMeasurementUnit" class="example-full-width" style="width: 155px" color="accent">
- <input matInput placeholder="Unit" disabled [(ngModel)]="oldMeasurementUnitDipsplay">
+ <mat-form-field *ngIf="hadMeasurementUnit"
+ class="example-full-width"
+ style="width: 155px"
+ color="accent">
+ <input matInput
+ placeholder="Unit"
+ disabled
+ [(ngModel)]="oldMeasurementUnitDipsplay"
+ data-cy="connect-schema-unit-from-input">
</mat-form-field>
</div>
- <button mat-button (click)="transformUnit()" color="accent" class="form-group" fxFlexAlign="center"
- style="min-width: 60px; max-width: 60px">
+ <button mat-button
+ (click)="transformUnit()"
+ color="accent"
+ class="form-group"
+ fxFlexAlign="center"
+ style="min-width: 60px; max-width: 60px"
+ data-cy="connect-schema-unit-transform-btn">
<mat-icon *ngIf="!transformUnitEnable">arrow_forward</mat-icon>
<mat-icon *ngIf="transformUnitEnable" style="transform: rotate(180deg)">arrow_forward</mat-icon>
</button>
<mat-form-field class="example-full-width" style="width: 155px" *ngIf="transformUnitEnable" color="accent">
- <mat-select placeholder="New Unit" [(ngModel)]="selectUnit" [formControl]="newUnitStateCtrl"[compareWith]="compareFn">
+ <mat-select placeholder="New Unit"
+ [(ngModel)]="selectUnit"
+ [formControl]="newUnitStateCtrl"
+ [compareWith]="compareFn"
+ data-cy="connect-schema-unit-to-dropdown">
<mat-option *ngFor="let unit of possibleUnitTransformations" [value]="unit"
(click)="changeTargetUnit(unit)">
{{unit.label}}