You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by bo...@apache.org on 2024/05/21 13:12:42 UTC
(streampipes) 02/02: fix(#2855): Issue with deletetransformationrule in adapter (#2887)
This is an automated email from the ASF dual-hosted git repository.
bossenti pushed a commit to branch rel/0.95.0
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 50971d787cc4fb55af8f9aa1f0b613780abfb1f0
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Tue May 21 15:11:52 2024 +0200
fix(#2855): Issue with deletetransformationrule in adapter (#2887)
* fix(#2855): Add e2e test to reproduce the bug
* fix(#2855): Fix transformation rules for properties with same prefix
---
.../transform/schema/DeleteTransformationRule.java | 2 +-
.../schema/DeleteTransformationRuleTest.java | 24 ++++---
.../deleteTransformationRule/nestedInput.json | 6 ++
.../deleteTransformationRule/prefixInput.csv | 2 +
.../support/utils/ConnectEventSchemaUtils.ts | 2 +-
ui/cypress/support/utils/connect/ConnectUtils.ts | 34 +++++-----
.../adapter/rules/deleteTransformationRule.spec.ts | 76 ++++++++++++++++++++++
.../event-schema-preview.component.html | 1 +
.../services/transformation-rule.service.spec.ts | 68 -------------------
.../services/transformation-rule.service.ts | 26 +-------
10 files changed, 119 insertions(+), 122 deletions(-)
diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java
index 11fb3abe4c..ae2d1c70d9 100644
--- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java
+++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java
@@ -38,6 +38,6 @@ public class DeleteTransformationRule extends SupportsNestedTransformationRule {
@Override
protected void applyTransformation(Map<String, Object> event, List<String> eventKeys) {
- event.remove(eventKeys.get(0));
+ event.remove(eventKeys.get(0));
}
}
diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java
index 0579efb627..7253b33bd5 100644
--- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java
+++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java
@@ -18,41 +18,39 @@
package org.apache.streampipes.connect.shared.preprocessing.transform.schema;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
public class DeleteTransformationRuleTest {
@Test
public void transformSimple() {
- Map<String, Object> event = new HashMap<>();
+ var event = new HashMap<String, Object>();
event.put("key", "value");
- DeleteTransformationRule deleteRule = new DeleteTransformationRule(List.of("key"));
+ var deleteRule = new DeleteTransformationRule(List.of("key"));
- Map<String, Object> result = deleteRule.apply(event);
+ var result = deleteRule.apply(event);
- Assertions.assertEquals(0,
- result.keySet().size());
+ assertEquals(0, result.keySet().size());
}
@Test
public void transformNested() {
- Map<String, Object> child = new HashMap<>();
+ var child = new HashMap<String, Object>();
child.put("child", "value");
- Map<String, Object> event = new HashMap<>();
+ var event = new HashMap<String, Object>();
event.put("parent", child);
- DeleteTransformationRule deleteRule = new DeleteTransformationRule(Arrays.asList("parent", "child"));
+ var deleteRule = new DeleteTransformationRule(Arrays.asList("parent", "child"));
- Map<String, Object> result = deleteRule.apply(event);
+ var result = deleteRule.apply(event);
- Assertions.assertEquals(1,
- result.keySet().size());
+ assertEquals(1, result.keySet().size());
}
}
diff --git a/ui/cypress/fixtures/connect/deleteTransformationRule/nestedInput.json b/ui/cypress/fixtures/connect/deleteTransformationRule/nestedInput.json
new file mode 100644
index 0000000000..92177c23ed
--- /dev/null
+++ b/ui/cypress/fixtures/connect/deleteTransformationRule/nestedInput.json
@@ -0,0 +1,6 @@
+{
+ "timestamp": 1667904471000,
+ "parent": {
+ "child": "text"
+ }
+}
diff --git a/ui/cypress/fixtures/connect/deleteTransformationRule/prefixInput.csv b/ui/cypress/fixtures/connect/deleteTransformationRule/prefixInput.csv
new file mode 100644
index 0000000000..5280fe3838
--- /dev/null
+++ b/ui/cypress/fixtures/connect/deleteTransformationRule/prefixInput.csv
@@ -0,0 +1,2 @@
+timestamp;reserved bit;reserved bit_1;reserved bit_2
+1715356080000;true;true;true
diff --git a/ui/cypress/support/utils/ConnectEventSchemaUtils.ts b/ui/cypress/support/utils/ConnectEventSchemaUtils.ts
index 1d933399ff..495a47c738 100644
--- a/ui/cypress/support/utils/ConnectEventSchemaUtils.ts
+++ b/ui/cypress/support/utils/ConnectEventSchemaUtils.ts
@@ -178,7 +178,7 @@ export class ConnectEventSchemaUtils {
}
public static deleteProperty(propertyName: string) {
- cy.dataCy('delete-property-' + propertyName, { timeout: 10000 })
+ cy.dataCy('"delete-property-' + propertyName + '"', { timeout: 10000 })
.children()
.click({ force: true });
cy.dataCy('connect-schema-delete-properties-btn', {
diff --git a/ui/cypress/support/utils/connect/ConnectUtils.ts b/ui/cypress/support/utils/connect/ConnectUtils.ts
index cfe06bfc6a..ed3b7bcb5b 100644
--- a/ui/cypress/support/utils/connect/ConnectUtils.ts
+++ b/ui/cypress/support/utils/connect/ConnectUtils.ts
@@ -276,29 +276,33 @@ export class ConnectUtils {
public static setUpPreprocessingRuleTest(
overwriteTimestamp: boolean,
+ adapterConfigurationBuilder?: AdapterBuilder,
): AdapterInput {
- const adapterConfiguration = AdapterBuilder.create('File_Stream')
- .setStoreInDataLake()
- .setTimestampProperty('timestamp')
- .addProtocolInput(
- 'radio',
- 'speed',
- 'fastest_\\(ignore_original_time\\)',
- )
- .addProtocolInput('radio', 'replayonce', 'yes')
- .setName('Adapter to test rules')
- .setFormat('csv')
- .addFormatInput('input', ConnectBtns.csvDelimiter(), ';')
- .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check');
+ if (!adapterConfigurationBuilder) {
+ adapterConfigurationBuilder = AdapterBuilder.create('File_Stream')
+ .setStoreInDataLake()
+ .setTimestampProperty('timestamp')
+ .addProtocolInput(
+ 'radio',
+ 'speed',
+ 'fastest_\\(ignore_original_time\\)',
+ )
+ .addProtocolInput('radio', 'replayonce', 'yes')
+ .setName('Adapter to test rules')
+ .setFormat('csv')
+ .addFormatInput('input', ConnectBtns.csvDelimiter(), ';')
+ .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check');
+ }
if (overwriteTimestamp) {
- adapterConfiguration.addProtocolInput(
+ adapterConfigurationBuilder.addProtocolInput(
'checkbox',
'replaceTimestamp',
'check',
);
}
- adapterConfiguration = adapterConfiguration.build();
+
+ const adapterConfiguration = adapterConfigurationBuilder.build();
ConnectUtils.goToConnect();
ConnectUtils.goToNewAdapterPage();
diff --git a/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts
new file mode 100644
index 0000000000..debbc5473f
--- /dev/null
+++ b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
+import { FileManagementUtils } from '../../../support/utils/FileManagementUtils';
+import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils';
+import { AdapterBuilder } from '../../../support/builder/AdapterBuilder';
+
+describe('Connect delete rule transformation', () => {
+ beforeEach('Setup Test', () => {
+ cy.initStreamPipesTest();
+ });
+
+ it('Test delete with same prefix', () => {
+ FileManagementUtils.addFile(
+ 'connect/deleteTransformationRule/prefixInput.csv',
+ );
+
+ ConnectUtils.setUpPreprocessingRuleTest(false);
+
+ ConnectEventSchemaUtils.deleteProperty('reserved bit');
+ ConnectEventSchemaUtils.deleteProperty('reserved bit_1');
+ ConnectEventSchemaUtils.deleteProperty('reserved bit_2');
+
+ cy.dataCy('schema-preview-result-event').should(
+ 'have.text',
+ '{\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01715356080000}',
+ );
+ });
+
+ it('Test delete nested properties', () => {
+ FileManagementUtils.addFile(
+ 'connect/deleteTransformationRule/nestedInput.json',
+ );
+
+ const adapterConfigurationBuilder = AdapterBuilder.create('File_Stream')
+ .setStoreInDataLake()
+ .setTimestampProperty('timestamp')
+ .addProtocolInput(
+ 'radio',
+ 'speed',
+ 'fastest_\\(ignore_original_time\\)',
+ )
+ .addProtocolInput('radio', 'replayonce', 'yes')
+ .setName('Adapter to test rules')
+ .setFormat('json');
+
+ ConnectUtils.setUpPreprocessingRuleTest(
+ false,
+ adapterConfigurationBuilder,
+ );
+
+ // Test to delete the child property
+ ConnectEventSchemaUtils.deleteProperty('child');
+
+ // The resulting string contains non-breaking spaces character (\u00A0)
+ cy.dataCy('schema-preview-result-event').should(
+ 'have.text',
+ '{\u00A0\u00A0\u00A0\u00A0"parent":\u00A0{},\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
+ );
+ });
+});
diff --git a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
index 41d8e5446c..41ae59c67b 100644
--- a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
+++ b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
@@ -49,6 +49,7 @@
<pre
[innerHTML]="desiredField | jsonpretty"
class="preview-text"
+ data-cy="schema-preview-result-event"
></pre>
</sp-basic-inner-panel>
</div>
diff --git a/ui/src/app/connect/services/transformation-rule.service.spec.ts b/ui/src/app/connect/services/transformation-rule.service.spec.ts
index 39c938b913..a0030bf34f 100644
--- a/ui/src/app/connect/services/transformation-rule.service.spec.ts
+++ b/ui/src/app/connect/services/transformation-rule.service.spec.ts
@@ -19,7 +19,6 @@
import { TransformationRuleService } from './transformation-rule.service';
import {
CreateNestedRuleDescription,
- DeleteRuleDescription,
EventPropertyNested,
EventPropertyPrimitive,
EventPropertyUnion,
@@ -196,73 +195,6 @@ describe('TransformationRuleService', () => {
expect(result.length).toBe(1);
expect(result[0].oldRuntimeKey).toBe('a');
- // expect(result[0].newRuntimeKey).toBe('b.a');
- });
-
- it('Delete simple', () => {
- const oldEventSchema: EventSchema = new EventSchema();
- const eventProperty: EventPropertyPrimitive =
- new EventPropertyPrimitive();
- eventProperty.runtimeName = 'a';
- oldEventSchema.eventProperties = [];
- oldEventSchema.eventProperties.push(eventProperty);
-
- const newEventSchema: EventSchema = new EventSchema();
- newEventSchema.eventProperties = [];
-
- const result: DeleteRuleDescription[] = service.getDeleteRules(
- newEventSchema.eventProperties,
- oldEventSchema,
- newEventSchema,
- );
-
- expect(result.length).toBe(1);
- expect(result[0].runtimeKey).toBe('a');
- });
-
- it('Delete nested', () => {
- const oldEventSchema: EventSchema = new EventSchema();
- const eventProperty: EventPropertyPrimitive =
- new EventPropertyPrimitive();
- eventProperty.elementId = 'id_2';
- eventProperty.runtimeName = 'a';
- const eventPropertyNested: EventPropertyNested =
- new EventPropertyNested();
- eventPropertyNested.elementId = 'id_1';
- eventPropertyNested.eventProperties = [];
- eventPropertyNested.eventProperties.push(eventProperty);
- eventPropertyNested.runtimeName = 'b';
- oldEventSchema.eventProperties = [];
- oldEventSchema.eventProperties.push(eventPropertyNested);
-
- let newEventSchema: EventSchema = new EventSchema();
- const newEventPropertyNested: EventPropertyNested =
- new EventPropertyNested();
- newEventPropertyNested.elementId = 'id_1';
- newEventPropertyNested.runtimeName = 'b';
- newEventPropertyNested.eventProperties = [];
- newEventSchema.eventProperties = [];
- newEventSchema.eventProperties.push(newEventPropertyNested);
-
- let result: DeleteRuleDescription[] = service.getDeleteRules(
- newEventSchema.eventProperties,
- oldEventSchema,
- newEventSchema,
- );
-
- expect(result.length).toBe(1);
- expect(result[0].runtimeKey).toBe('b.a');
-
- newEventSchema = new EventSchema();
- newEventSchema.eventProperties = [];
- result = service.getDeleteRules(
- newEventSchema.eventProperties,
- oldEventSchema,
- newEventSchema,
- );
-
- expect(result.length).toBe(1);
- expect(result[0].runtimeKey).toBe('b');
});
it('Rename simple', () => {
diff --git a/ui/src/app/connect/services/transformation-rule.service.ts b/ui/src/app/connect/services/transformation-rule.service.ts
index f1cca8565c..b0fd79fa3d 100644
--- a/ui/src/app/connect/services/transformation-rule.service.ts
+++ b/ui/src/app/connect/services/transformation-rule.service.ts
@@ -125,13 +125,7 @@ export class TransformationRuleService {
targetSchema,
),
)
- .concat(
- this.getDeleteRules(
- targetSchema.eventProperties,
- originalSchema,
- targetSchema,
- ),
- )
+ .concat(this.getDeleteRules(originalSchema, targetSchema))
.concat(
this.getUnitTransformRules(
targetSchema.eventProperties,
@@ -299,11 +293,10 @@ export class TransformationRuleService {
}
public getDeleteRules(
- newEventProperties: EventProperty[],
oldEventSchema: EventSchema,
newEventSchema: EventSchema,
): DeleteRuleDescription[] {
- let resultKeys: string[] = [];
+ const resultKeys: string[] = [];
const allNewIds: string[] = this.getAllIds(
newEventSchema.eventProperties,
@@ -323,21 +316,6 @@ export class TransformationRuleService {
}
}
- const uniqEs6 = arrArg => {
- return arrArg.filter((elem, pos, arr) => {
- let r = true;
- for (const a of arr) {
- if (elem.startsWith(a) && a !== elem) {
- r = false;
- }
- }
-
- return r;
- });
- };
-
- resultKeys = uniqEs6(resultKeys);
-
const resultRules: DeleteRuleDescription[] = [];
for (const key of resultKeys) {
const rule: DeleteRuleDescription = new DeleteRuleDescription();