You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/01/09 10:25:46 UTC
[streampipes] branch SP-1065 created (now 3123212a7)
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a change to branch SP-1065
in repository https://gitbox.apache.org/repos/asf/streampipes.git
at 3123212a7 [#1065] Add e2e tests for all json formats
This branch includes the following new commits:
new 3123212a7 [#1065] Add e2e tests for all json formats
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[streampipes] 01/01: [#1065] Add e2e tests for all json formats
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch SP-1065
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 3123212a72894e32d56456fcd6dfcdbdddf8ed02
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Mon Jan 9 11:25:19 2023 +0100
[#1065] Add e2e tests for all json formats
---
.../adapter/format/geojson/GeoJsonParser.java | 58 ++++++---
...onObjectParser.java => AbstractJsonParser.java} | 49 +-------
.../adapter/format/json/arraykey/JsonParser.java | 32 +----
.../format/json/arraynokey/JsonArrayParser.java | 36 +-----
.../format/json/object/JsonObjectParser.java | 47 +------
.../fixtures/connect/formatTests/geoJson.json | 16 +++
.../fixtures/connect/formatTests/jsonArray.json | 9 ++
.../connect/formatTests/jsonArrayField.json | 11 ++
.../fixtures/connect/formatTests/jsonObject.json | 7 ++
ui/cypress/support/utils/connect/ConnectBtns.ts | 43 +++++++
ui/cypress/support/utils/connect/ConnectUtils.ts | 12 +-
.../adapter/formats/jsonObjectFormat.smoke.spec.ts | 139 +++++++++++++++++++++
.../event-schema-preview.component.html | 6 +-
13 files changed, 285 insertions(+), 180 deletions(-)
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/geojson/GeoJsonParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/geojson/GeoJsonParser.java
index f09a8405d..632f93a00 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/geojson/GeoJsonParser.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/geojson/GeoJsonParser.java
@@ -24,6 +24,8 @@ import org.apache.streampipes.extensions.api.connect.exception.ParseException;
import org.apache.streampipes.extensions.management.connect.adapter.format.util.JsonEventProperty;
import org.apache.streampipes.extensions.management.connect.adapter.model.generic.Parser;
import org.apache.streampipes.model.connect.grounding.FormatDescription;
+import org.apache.streampipes.model.connect.guess.AdapterGuessInfo;
+import org.apache.streampipes.model.connect.guess.GuessTypeInfo;
import org.apache.streampipes.model.schema.EventProperty;
import org.apache.streampipes.model.schema.EventPropertyPrimitive;
import org.apache.streampipes.model.schema.EventSchema;
@@ -34,7 +36,6 @@ import com.google.common.base.Charsets;
import com.google.common.io.CharStreams;
import com.google.gson.Gson;
import org.geojson.Feature;
-import org.geojson.FeatureCollection;
import org.geojson.LineString;
import org.geojson.MultiLineString;
import org.geojson.MultiPoint;
@@ -66,7 +67,6 @@ public class GeoJsonParser extends Parser {
@Override
public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) throws ParseException {
- FeatureCollection geoFeature;
Gson gson = new Gson();
try {
@@ -85,29 +85,33 @@ public class GeoJsonParser extends Parser {
@Override
public EventSchema getEventSchema(List<byte[]> oneEvent) {
- EventSchema resultSchema = new EventSchema();
+ return this.getSchemaAndSample(oneEvent).getEventSchema();
+ }
+
+ @Override
+ public boolean supportsPreview() {
+ return true;
+ }
+
+ @Override
+ public AdapterGuessInfo getSchemaAndSample(List<byte[]> eventSample) throws ParseException {
Feature geoFeature = null;
try {
- geoFeature = new ObjectMapper().readValue(oneEvent.get(0), Feature.class);
+ geoFeature = new ObjectMapper().readValue(eventSample.get(0), Feature.class);
} catch (IOException e) {
logger.error(e.toString());
}
- for (Map.Entry<String, Object> entry : geoFeature.getProperties().entrySet()) {
- EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue());
- resultSchema.addEventProperty(p);
- }
-
- List<EventProperty> eventProperties = parseGeometryField(geoFeature);
- eventProperties.forEach(eventProperty -> resultSchema.addEventProperty(eventProperty));
-
- return resultSchema;
+ return this.parseGeometryField(geoFeature);
}
- private List<EventProperty> parseGeometryField(Feature geoFeature) {
+ private AdapterGuessInfo parseGeometryField(Feature geoFeature) {
+
+ EventSchema resultSchema = new EventSchema();
List<EventProperty> eventProperties = new LinkedList<>();
+ var sampleValues = new HashMap<String, GuessTypeInfo>();
if (geoFeature.getGeometry() instanceof Point) {
Point point = (Point) geoFeature.getGeometry();
@@ -115,35 +119,51 @@ public class GeoJsonParser extends Parser {
"http://www.w3.org/2003/01/geo/wgs84_pos#long"));
eventProperties.add(getEventPropertyGeoJson("latitude", point.getCoordinates().getLatitude(),
"http://www.w3.org/2003/01/geo/wgs84_pos#lat"));
+
+ sampleValues.put("longitude", new GuessTypeInfo("java.lang.Double", point.getCoordinates().getLongitude()));
+ sampleValues.put("latitude", new GuessTypeInfo("java.lang.Double", point.getCoordinates().getLatitude()));
if (point.getCoordinates().hasAltitude()) {
eventProperties.add(getEventPropertyGeoJson("altitude", point.getCoordinates().getAltitude(), SO.ALTITUDE));
+ sampleValues.put("altitude", new GuessTypeInfo("java.lang.Double", point.getCoordinates().getAltitude()));
}
} else if (geoFeature.getGeometry() instanceof LineString) {
LineString lineString = (LineString) geoFeature.getGeometry();
eventProperties.add(JsonEventProperty.getEventProperty("coorindatesLineString", lineString.getCoordinates()));
-
+ sampleValues.put("coorindatesLineString", new GuessTypeInfo("java.lang.Double", lineString.getCoordinates()));
} else if (geoFeature.getGeometry() instanceof Polygon) {
Polygon polygon = (Polygon) geoFeature.getGeometry();
eventProperties.add(JsonEventProperty.getEventProperty("coorindatesPolygon", polygon.getCoordinates()));
-
+ sampleValues.put("coorindatesPolygon", new GuessTypeInfo("java.lang.Double", polygon.getCoordinates()));
} else if (geoFeature.getGeometry() instanceof MultiPoint) {
MultiPoint multiPoint = (MultiPoint) geoFeature.getGeometry();
eventProperties.add(JsonEventProperty.getEventProperty("coorindatesMultiPoint", multiPoint.getCoordinates()));
-
+ sampleValues.put("coorindatesMultiPoint", new GuessTypeInfo("java.lang.Double", multiPoint.getCoordinates()));
} else if (geoFeature.getGeometry() instanceof MultiLineString) {
MultiLineString multiLineString = (MultiLineString) geoFeature.getGeometry();
eventProperties.add(
JsonEventProperty.getEventProperty("coorindatesMultiLineString", multiLineString.getCoordinates()));
-
+ sampleValues.put("coorindatesMultiLineString",
+ new GuessTypeInfo("java.lang.Double", multiLineString.getCoordinates()));
} else if (geoFeature.getGeometry() instanceof MultiPolygon) {
MultiPolygon multiPolygon = (MultiPolygon) geoFeature.getGeometry();
eventProperties.add(JsonEventProperty.getEventProperty("coorindatesMultiPolygon", multiPolygon.getCoordinates()));
+ sampleValues.put("coorindatesMultiPolygon", new GuessTypeInfo("java.lang.Double", multiPolygon.getCoordinates()));
} else {
logger.error("No geometry field found in geofeature: " + geoFeature.toString());
}
- return eventProperties;
+
+ for (Map.Entry<String, Object> entry : geoFeature.getProperties().entrySet()) {
+ EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue());
+ resultSchema.addEventProperty(p);
+ sampleValues.put(p.getRuntimeName(),
+ new GuessTypeInfo(entry.getValue().getClass().getCanonicalName(), entry.getValue()));
+ }
+
+ eventProperties.forEach(eventProperty -> resultSchema.addEventProperty(eventProperty));
+
+ return new AdapterGuessInfo(resultSchema, sampleValues);
}
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/AbstractJsonParser.java
similarity index 67%
copy from streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java
copy to streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/AbstractJsonParser.java
index 58efc8e83..3d04e882f 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/AbstractJsonParser.java
@@ -16,65 +16,23 @@
*
*/
-package org.apache.streampipes.extensions.management.connect.adapter.format.json.object;
-
+package org.apache.streampipes.extensions.management.connect.adapter.format.json;
import org.apache.streampipes.commons.exceptions.SpRuntimeException;
import org.apache.streampipes.dataformat.json.JsonDataFormatDefinition;
-import org.apache.streampipes.extensions.api.connect.EmitBinaryEvent;
import org.apache.streampipes.extensions.api.connect.exception.ParseException;
import org.apache.streampipes.extensions.management.connect.adapter.format.util.JsonEventProperty;
import org.apache.streampipes.extensions.management.connect.adapter.model.generic.Parser;
-import org.apache.streampipes.model.connect.grounding.FormatDescription;
import org.apache.streampipes.model.connect.guess.AdapterGuessInfo;
import org.apache.streampipes.model.connect.guess.GuessTypeInfo;
import org.apache.streampipes.model.schema.EventProperty;
import org.apache.streampipes.model.schema.EventSchema;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-public class JsonObjectParser extends Parser {
-
- private static final Logger logger = LoggerFactory.getLogger(JsonObjectParser.class);
-
- private ObjectMapper mapper;
- private JsonDataFormatDefinition jsonDefinition;
-
- @Override
- public Parser getInstance(FormatDescription formatDescription) {
- return new JsonObjectParser();
- }
-
- /**
- * Use this constructor when just a specific key of the object should be parsed
- */
- public JsonObjectParser() {
- mapper = new ObjectMapper();
- jsonDefinition = new JsonDataFormatDefinition();
- }
-
- @Override
- public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) throws ParseException {
-
- try {
- Map<String, Object> map = mapper.readValue(data, HashMap.class);
- emitBinaryEvent.emit(jsonDefinition.fromMap(map));
- } catch (IOException e) {
- e.printStackTrace();
- } catch (SpRuntimeException e) {
- e.printStackTrace();
- }
-
- }
+public abstract class AbstractJsonParser extends Parser {
@Override
public EventSchema getEventSchema(List<byte[]> oneEvent) {
@@ -93,7 +51,7 @@ public class JsonObjectParser extends Parser {
JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition();
- Map<String, Object> exampleEvent = null;
+ Map<String, Object> exampleEvent;
try {
exampleEvent = jsonDefinition.toMap(eventSample.get(0));
@@ -114,4 +72,5 @@ public class JsonObjectParser extends Parser {
throw new ParseException("Could not serialize event, did you choose the correct format?", e);
}
}
+
}
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraykey/JsonParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraykey/JsonParser.java
index 5f1b31dcb..0193ab936 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraykey/JsonParser.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraykey/JsonParser.java
@@ -18,17 +18,14 @@
package org.apache.streampipes.extensions.management.connect.adapter.format.json.arraykey;
-
import org.apache.streampipes.commons.exceptions.SpRuntimeException;
import org.apache.streampipes.dataformat.json.JsonDataFormatDefinition;
import org.apache.streampipes.extensions.api.connect.EmitBinaryEvent;
import org.apache.streampipes.extensions.api.connect.exception.ParseException;
-import org.apache.streampipes.extensions.management.connect.adapter.format.util.JsonEventProperty;
+import org.apache.streampipes.extensions.management.connect.adapter.format.json.AbstractJsonParser;
import org.apache.streampipes.extensions.management.connect.adapter.model.generic.Parser;
import org.apache.streampipes.extensions.management.connect.adapter.sdk.ParameterExtractor;
import org.apache.streampipes.model.connect.grounding.FormatDescription;
-import org.apache.streampipes.model.schema.EventProperty;
-import org.apache.streampipes.model.schema.EventSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,7 +39,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class JsonParser extends Parser {
+public class JsonParser extends AbstractJsonParser {
Logger logger = LoggerFactory.getLogger(JsonParser.class);
private boolean isArray;
@@ -121,31 +118,6 @@ public class JsonParser extends Parser {
}
}
- @Override
- public EventSchema getEventSchema(List<byte[]> oneEvent) {
- EventSchema resultSchema = new EventSchema();
-
- JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition();
-
- Map<String, Object> exampleEvent = null;
-
- try {
- exampleEvent = jsonDefinition.toMap(oneEvent.get(0));
- } catch (SpRuntimeException e) {
- e.printStackTrace();
- }
-
- for (Map.Entry<String, Object> entry : exampleEvent.entrySet()) {
-// System.out.println(entry.getKey() + "/" + entry.getValue());
- EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue());
-
- resultSchema.addEventProperty(p);
-
- }
-
- return resultSchema;
- }
-
public Map<String, Object> parseObject(jakarta.json.stream.JsonParser jsonParser, boolean root, int start) {
// this variable is needed to skip the first object start
String mapKey = "";
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraynokey/JsonArrayParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraynokey/JsonArrayParser.java
index b028563eb..f0baa3eb7 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraynokey/JsonArrayParser.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraynokey/JsonArrayParser.java
@@ -23,11 +23,9 @@ import org.apache.streampipes.commons.exceptions.SpRuntimeException;
import org.apache.streampipes.dataformat.json.JsonDataFormatDefinition;
import org.apache.streampipes.extensions.api.connect.EmitBinaryEvent;
import org.apache.streampipes.extensions.api.connect.exception.ParseException;
-import org.apache.streampipes.extensions.management.connect.adapter.format.util.JsonEventProperty;
+import org.apache.streampipes.extensions.management.connect.adapter.format.json.AbstractJsonParser;
import org.apache.streampipes.extensions.management.connect.adapter.model.generic.Parser;
import org.apache.streampipes.model.connect.grounding.FormatDescription;
-import org.apache.streampipes.model.schema.EventProperty;
-import org.apache.streampipes.model.schema.EventSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,7 +40,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class JsonArrayParser extends Parser {
+public class JsonArrayParser extends AbstractJsonParser {
Logger logger = LoggerFactory.getLogger(JsonArrayParser.class);
@@ -58,7 +56,6 @@ public class JsonArrayParser extends Parser {
@Override
public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) throws ParseException {
JsonParserFactory factory = Json.createParserFactory(null);
- String s = data.toString();
JsonParser jsonParser = factory.createParser(data);
// Find first event in array
@@ -77,11 +74,10 @@ public class JsonArrayParser extends Parser {
JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition();
boolean isEvent = true;
boolean result = true;
- int objectCount = 0;
while (jsonParser.hasNext() && isEvent && result) {
Map<String, Object> objectMap = parseObject(jsonParser, true, 1);
if (objectMap != null) {
- byte[] tmp = new byte[0];
+ byte[] tmp;
try {
tmp = jsonDefinition.fromMap(objectMap);
} catch (SpRuntimeException e) {
@@ -97,32 +93,6 @@ public class JsonArrayParser extends Parser {
}
}
- @Override
- public EventSchema getEventSchema(List<byte[]> oneEvent) {
- EventSchema resultSchema = new EventSchema();
-
- JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition();
-
- Map<String, Object> exampleEvent = null;
-
- try {
- exampleEvent = jsonDefinition.toMap(oneEvent.get(0));
- } catch (SpRuntimeException e) {
- e.printStackTrace();
- }
-
- for (Map.Entry<String, Object> entry : exampleEvent.entrySet()) {
-// System.out.println(entry.getKey() + "/" + entry.getValue());
- EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue());
-
- resultSchema.addEventProperty(p);
-
- }
-
- return resultSchema;
- }
-
-
public Map<String, Object> parseObject(JsonParser jsonParser, boolean root, int start) {
// this variable is needed to skip the first object start
String mapKey = "";
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java
index 58efc8e83..02a186dea 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java
@@ -23,13 +23,9 @@ import org.apache.streampipes.commons.exceptions.SpRuntimeException;
import org.apache.streampipes.dataformat.json.JsonDataFormatDefinition;
import org.apache.streampipes.extensions.api.connect.EmitBinaryEvent;
import org.apache.streampipes.extensions.api.connect.exception.ParseException;
-import org.apache.streampipes.extensions.management.connect.adapter.format.util.JsonEventProperty;
+import org.apache.streampipes.extensions.management.connect.adapter.format.json.AbstractJsonParser;
import org.apache.streampipes.extensions.management.connect.adapter.model.generic.Parser;
import org.apache.streampipes.model.connect.grounding.FormatDescription;
-import org.apache.streampipes.model.connect.guess.AdapterGuessInfo;
-import org.apache.streampipes.model.connect.guess.GuessTypeInfo;
-import org.apache.streampipes.model.schema.EventProperty;
-import org.apache.streampipes.model.schema.EventSchema;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
@@ -38,11 +34,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
-import java.util.stream.Collectors;
-public class JsonObjectParser extends Parser {
+public class JsonObjectParser extends AbstractJsonParser {
private static final Logger logger = LoggerFactory.getLogger(JsonObjectParser.class);
@@ -76,42 +70,5 @@ public class JsonObjectParser extends Parser {
}
- @Override
- public EventSchema getEventSchema(List<byte[]> oneEvent) {
- return getSchemaAndSample(oneEvent).getEventSchema();
- }
-
- @Override
- public boolean supportsPreview() {
- return true;
- }
-
- @Override
- public AdapterGuessInfo getSchemaAndSample(List<byte[]> eventSample) throws ParseException {
- EventSchema resultSchema = new EventSchema();
-
- JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition();
-
- Map<String, Object> exampleEvent = null;
-
- try {
- exampleEvent = jsonDefinition.toMap(eventSample.get(0));
- var sample = exampleEvent
- .entrySet()
- .stream()
- .collect(Collectors.toMap(Map.Entry::getKey, e ->
- new GuessTypeInfo(e.getValue().getClass().getCanonicalName(), e.getValue())));
-
- for (Map.Entry<String, Object> entry : exampleEvent.entrySet()) {
- EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue());
-
- resultSchema.addEventProperty(p);
- }
-
- return new AdapterGuessInfo(resultSchema, sample);
- } catch (SpRuntimeException e) {
- throw new ParseException("Could not serialize event, did you choose the correct format?", e);
- }
- }
}
diff --git a/ui/cypress/fixtures/connect/formatTests/geoJson.json b/ui/cypress/fixtures/connect/formatTests/geoJson.json
new file mode 100644
index 000000000..047e60497
--- /dev/null
+++ b/ui/cypress/fixtures/connect/formatTests/geoJson.json
@@ -0,0 +1,16 @@
+{
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "geometry": {
+ "type": "Point",
+ "coordinates": [125.6, 10.1]
+ },
+ "properties": {
+ "timestamp": 1667904471000,
+ "v1": 4.1
+ }
+ }
+ ]
+}
diff --git a/ui/cypress/fixtures/connect/formatTests/jsonArray.json b/ui/cypress/fixtures/connect/formatTests/jsonArray.json
new file mode 100644
index 000000000..db655b79d
--- /dev/null
+++ b/ui/cypress/fixtures/connect/formatTests/jsonArray.json
@@ -0,0 +1,9 @@
+[
+ {
+ "timestamp": 1667904471000,
+ "v1": 4.1,
+ "v2": "abc",
+ "v3": true,
+ "v4": 1
+ }
+]
diff --git a/ui/cypress/fixtures/connect/formatTests/jsonArrayField.json b/ui/cypress/fixtures/connect/formatTests/jsonArrayField.json
new file mode 100644
index 000000000..efa2ec775
--- /dev/null
+++ b/ui/cypress/fixtures/connect/formatTests/jsonArrayField.json
@@ -0,0 +1,11 @@
+{
+ "field": [
+ {
+ "timestamp": 1667904471000,
+ "v1": 4.1,
+ "v2": "abc",
+ "v3": true,
+ "v4": 1
+ }
+ ]
+}
diff --git a/ui/cypress/fixtures/connect/formatTests/jsonObject.json b/ui/cypress/fixtures/connect/formatTests/jsonObject.json
new file mode 100644
index 000000000..48b82b791
--- /dev/null
+++ b/ui/cypress/fixtures/connect/formatTests/jsonObject.json
@@ -0,0 +1,7 @@
+{
+ "timestamp": 1667904471000,
+ "v1": 4.1,
+ "v2": "abc",
+ "v3": true,
+ "v4": 1
+}
diff --git a/ui/cypress/support/utils/connect/ConnectBtns.ts b/ui/cypress/support/utils/connect/ConnectBtns.ts
index e79089fa0..17887ff2a 100644
--- a/ui/cypress/support/utils/connect/ConnectBtns.ts
+++ b/ui/cypress/support/utils/connect/ConnectBtns.ts
@@ -39,4 +39,47 @@ export class ConnectBtns {
public static storeEditAdapter() {
return cy.dataCy('store-edit-adapter');
}
+
+ public static formatSelectionNextBtn() {
+ return cy
+ .dataCy('sp-format-selection-next-button')
+ .contains('Next')
+ .parent();
+ }
+
+ // =================================================================
+ // Format button options
+ public static json() {
+ return cy.dataCy('connect-select-json-formats');
+ }
+
+ public static jsonArrayField() {
+ return cy.dataCy('array_field');
+ }
+
+ public static geoJson() {
+ return cy.dataCy('geojson');
+ }
+
+ public static jsonObject() {
+ return cy.dataCy('single_object');
+ }
+
+ public static jsonArray() {
+ return cy.dataCy('array');
+ }
+
+ public static xml() {
+ return cy.dataCy('xml');
+ }
+
+ public static csv() {
+ return cy.dataCy('csv');
+ }
+
+ public static image() {
+ return cy.dataCy('image');
+ }
+
+ // =================================================================
}
diff --git a/ui/cypress/support/utils/connect/ConnectUtils.ts b/ui/cypress/support/utils/connect/ConnectUtils.ts
index 8b77258b2..feca0e004 100644
--- a/ui/cypress/support/utils/connect/ConnectUtils.ts
+++ b/ui/cypress/support/utils/connect/ConnectUtils.ts
@@ -25,6 +25,7 @@ import { AdapterInput } from '../../model/AdapterInput';
import { ConnectEventSchemaUtils } from '../ConnectEventSchemaUtils';
import { GenericAdapterBuilder } from '../../builder/GenericAdapterBuilder';
import { DataLakeUtils } from '../datalake/DataLakeUtils';
+import { ConnectBtns } from './ConnectBtns';
export class ConnectUtils {
public static testSpecificStreamAdapter(
@@ -163,11 +164,11 @@ export class ConnectUtils {
public static configureFormat(adapterConfiguration: GenericAdapterInput) {
// Select format
if (adapterConfiguration.format.indexOf('json') !== -1) {
- cy.dataCy('connect-select-json-formats').click();
+ ConnectBtns.json().click();
if (adapterConfiguration.format.indexOf('object') !== -1) {
- cy.dataCy('single_object').click();
+ ConnectBtns.jsonObject().click();
} else {
- cy.dataCy('array').click();
+ ConnectBtns.jsonArray().click();
}
} else {
cy.dataCy(adapterConfiguration.format).click();
@@ -176,10 +177,7 @@ export class ConnectUtils {
StaticPropertyUtils.input(adapterConfiguration.formatConfiguration);
// Click next
- cy.dataCy('sp-format-selection-next-button')
- .contains('Next')
- .parent()
- .click();
+ ConnectBtns.formatSelectionNextBtn().click();
}
public static finishEventSchemaConfiguration() {
diff --git a/ui/cypress/tests/adapter/formats/jsonObjectFormat.smoke.spec.ts b/ui/cypress/tests/adapter/formats/jsonObjectFormat.smoke.spec.ts
new file mode 100644
index 000000000..cc8d40c38
--- /dev/null
+++ b/ui/cypress/tests/adapter/formats/jsonObjectFormat.smoke.spec.ts
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { FileManagementUtils } from '../../../support/utils/FileManagementUtils';
+import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
+import { ConnectBtns } from '../../../support/utils/connect/ConnectBtns';
+import { StaticPropertyUtils } from '../../../support/utils/StaticPropertyUtils';
+import { UserInputBuilder } from '../../../support/builder/UserInputBuilder';
+
+describe('Test adapter formats', () => {
+ beforeEach('Setup Test', () => {
+ cy.initStreamPipesTest();
+ });
+
+ const expected = {
+ timestamp: 1667904471000,
+ v1: 4.1,
+ v2: 'abc',
+ v3: true,
+ v4: 1,
+ };
+
+ it('Test json object format', () => {
+ // Set up test
+ FileManagementUtils.addFile('connect/formatTests/jsonObject.json');
+
+ navigateToFormatSelection();
+
+ // Set format configuration
+ ConnectBtns.json().click();
+ ConnectBtns.jsonObject().click();
+
+ // Validate result
+ validateResult(expected);
+ });
+
+ it('Test array with json objects', () => {
+ // Set up test
+ FileManagementUtils.addFile('connect/formatTests/jsonArray.json');
+ navigateToFormatSelection();
+
+ // Set format configuration
+ ConnectBtns.json().click();
+ ConnectBtns.jsonArray().click();
+
+ // Validate result
+ validateResult(expected);
+ });
+
+ it('Test json with a field of type array', () => {
+ // Set up test
+ FileManagementUtils.addFile('connect/formatTests/jsonArrayField.json');
+ navigateToFormatSelection();
+
+ // Set format configuration
+ ConnectBtns.json().click();
+ ConnectBtns.jsonArrayField().click();
+ const arrayFieldInput = UserInputBuilder.create()
+ .add('input', 'key', 'field')
+ .build();
+ StaticPropertyUtils.input(arrayFieldInput);
+
+ // Validate result
+ validateResult(expected);
+ });
+
+ it('Test geo json format', () => {
+ // Set up test
+ const geoJsonResultEvent = {
+ latitude: 10.1,
+ longitude: 125.6,
+ timestamp: 1667904471000,
+ v1: 4.1,
+ };
+ FileManagementUtils.addFile('connect/formatTests/geoJson.json');
+ navigateToFormatSelection();
+
+ // Set format configuration
+ ConnectBtns.json().click();
+ ConnectBtns.geoJson().click();
+
+ // Validate result
+ validateResult(geoJsonResultEvent);
+ });
+});
+
+const navigateToFormatSelection = () => {
+ ConnectUtils.goToConnect();
+
+ ConnectUtils.goToNewAdapterPage();
+
+ ConnectUtils.selectAdapter('File_Set');
+
+ ConnectUtils.configureAdapter([]);
+};
+
+const validateResult = expected => {
+ ConnectBtns.formatSelectionNextBtn().click();
+ cy.dataCy('schema-preview-original-event', { timeout: 10000 }).then(
+ value => {
+ const jsonResult = removeWhitespaceExceptInQuotes(value.text());
+ expect(jsonResult).to.deep.equal(expected);
+ },
+ );
+};
+
+const removeWhitespaceExceptInQuotes = (input: string): string => {
+ let inQuote = false;
+ let result = '';
+
+ for (const char of input) {
+ if (char === '"') {
+ inQuote = !inQuote;
+ }
+
+ if (/\s/.test(char) && !inQuote) {
+ continue;
+ }
+
+ result += char;
+ }
+
+ return JSON.parse(result);
+};
diff --git a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
index d3cad53f6..6910eb9c7 100644
--- a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
+++ b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
@@ -19,7 +19,11 @@
<div fxFlex="100" fxLayout="row" fxLayoutGap="15px">
<div fxFlex="50" fxLayout="column">
<sp-basic-inner-panel [showTitle]="true" panelTitle="Original (Parsed)">
- <pre [innerHTML]="originalField | jsonpretty" class="preview-text"></pre>
+ <pre
+ [innerHTML]="originalField | jsonpretty"
+ class="preview-text"
+ data-cy="schema-preview-original-event"
+ ></pre>
</sp-basic-inner-panel>
</div>