You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/01/13 13:12:36 UTC
[streampipes] branch SP-1085 updated (d9250f91f -> 02059a87d)
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a change to branch SP-1085
in repository https://gitbox.apache.org/repos/asf/streampipes.git
from d9250f91f Merge branch 'dev' into SP-1085
add 3123212a7 [#1065] Add e2e tests for all json formats
add 5a2a19d39 Merge branch 'dev' into SP-1065
add c36c21fce [#1065] Replace library underscore with jackson-dataformat-xml
add 898ad6c0f [SP-1065] Add e2e test for xml format
add 609b3f379 [SP-1065] Add e2e test for csv format
add 80afe5ecb [hotfix] Add missing header and fix dependency convergence
add ce0b5cafc [#1065] Extracted common strings into GeoJsonConstants
add 3854af17a Bump konva from 8.3.14 to 8.4.0 in /ui (#1083)
add 5119886c6 Bump simple-java-mail from 7.5.2 to 7.6.0 (#1076)
add 1e66a8b8d [#877] Format and lint core module (#1086)
add 1cde20aba Remove unused classes (#1058)
add 8a1161839 Remove obsolete event property quality classes (#1058)
add c59017c13 Eliminate Abstract and UnnamedStreamPipesEntity classes (#SP-1058)
add 7b17ffe60 Bump jgrapht-core from 1.3.1 to 1.5.1
add 4e61e6848 Merge pull request #1075 from apache/dependabot/maven/org.jgrapht-jgrapht-core-1.5.1
add 8de82671b Bump protobuf-java from 3.16.3 to 3.21.12
add 505fdb6d0 Merge pull request #1074 from apache/dependabot/maven/com.google.protobuf-protobuf-java-3.21.12
add 8d07cafd2 Merge branch 'dev' into SP-1065
new 77354bffc Merge remote-tracking branch 'origin/SP-1065' into SP-1085
new 02059a87d [#1085] Extract find timestamp property from FileStreamProtocol to EventSchemaUtils
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
pom.xml | 12 +-
.../api/declarer/EventStreamDeclarer.java | 25 -
.../declarer/SemanticEventProducerDeclarer.java | 28 -
streampipes-extensions-management/pom.xml | 20 +-
.../connect/adapter/AdapterPipelineGenerator.java | 9 +-
.../connect/adapter/format/csv/CsvParser.java | 4 +-
.../geojson/GeoJsonConstants.java} | 30 +-
.../adapter/format/geojson/GeoJsonFormat.java | 28 +-
.../adapter/format/geojson/GeoJsonParser.java | 99 +-
...onObjectParser.java => AbstractJsonParser.java} | 49 +-
.../adapter/format/json/arraykey/JsonParser.java | 32 +-
.../format/json/arraynokey/JsonArrayParser.java | 36 +-
.../format/json/object/JsonObjectParser.java | 47 +-
.../adapter/format/xml/XmlMapConverter.java | 20 +-
.../connect/adapter/format/xml/XmlParser.java | 37 +-
.../adapter/model/generic/GenericAdapter.java | 4 -
.../pipeline/AdapterEventPreviewPipeline.java | 6 +
.../adapter/model/pipeline/AdapterPipeline.java | 8 +-
.../management/util/EventSchemaUtils.java | 29 +-
.../adapter/format/xml/XmlMapConverterTest.java | 37 +-
.../management/util/EventSchemaUtilsTest.java | 18 +-
.../streampipes-connect-adapters-iiot/pom.xml | 6 +
.../iiot/protocol/stream/FileStreamProtocol.java | 34 +-
streampipes-model/pom.xml | 5 -
.../apache/streampipes/model/ApplicationLink.java | 95 --
.../org/apache/streampipes/model/SpDataSet.java | 11 +-
.../org/apache/streampipes/model/SpDataStream.java | 67 -
.../streampipes/model/SpDataStreamContainer.java | 8 +-
.../model/base/AbstractStreamPipesEntity.java | 71 -
.../model/base/NamedStreamPipesEntity.java | 50 +-
.../apache/streampipes/model/base/Namespaces.java | 25 -
.../model/base/UnnamedStreamPipesEntity.java | 45 -
.../model/connect/guess/GuessSchema.java | 7 +-
.../rules/TransformationRuleDescription.java | 9 +-
.../model/dashboard/DashboardWidgetSettings.java | 3 +-
.../model/datalake/DataLakeMeasure.java | 16 +-
.../model/graph/DataSourceDescription.java | 93 --
.../model/grounding/EventGrounding.java | 4 +-
.../model/grounding/TopicDefinition.java | 7 +-
.../model/grounding/TransportFormat.java | 5 +-
.../model/grounding/TransportProtocol.java | 18 +-
.../model/grounding/WildcardTopicMapping.java | 4 +-
.../monitoring/ElementStatusInfoSettings.java | 5 +-
.../streampipes/model/output/OutputStrategy.java | 6 +-
.../model/output/PropertyRenameRule.java | 5 +-
.../model/output/TransformOperation.java | 5 +-
.../apache/streampipes/model/quality/Accuracy.java | 47 -
.../quality/EventPropertyQualityDefinition.java | 41 -
.../quality/EventPropertyQualityRequirement.java | 82 --
.../quality/EventStreamQualityDefinition.java | 40 -
.../quality/EventStreamQualityRequirement.java | 64 -
.../streampipes/model/quality/Frequency.java | 52 -
.../apache/streampipes/model/quality/Latency.java | 63 -
.../model/quality/MeasurementCapability.java | 54 -
.../model/quality/MeasurementObject.java | 54 -
.../model/quality/MeasurementProperty.java | 41 -
.../model/quality/MeasurementRange.java | 61 -
.../streampipes/model/quality/Precision.java | 48 -
.../streampipes/model/quality/Resolution.java | 48 -
.../model/runtime/RuntimeOptionsRequest.java | 6 +-
.../streampipes/model/schema/Enumeration.java | 1 -
.../streampipes/model/schema/EventProperty.java | 69 +-
.../model/schema/EventPropertyPrimitive.java | 10 -
.../streampipes/model/schema/EventSchema.java | 4 +-
.../model/schema/QuantitativeValue.java | 3 -
.../model/schema/ValueSpecification.java | 10 +-
.../streampipes/model/staticproperty/Option.java | 26 +-
.../staticproperty/PropertyValueSpecification.java | 4 +-
.../model/staticproperty/StaticProperty.java | 6 +-
.../staticproperty/StaticPropertyAlternative.java | 13 +
.../model/staticproperty/SupportedProperty.java | 4 +-
.../model/template/BoundPipelineElement.java | 5 +-
.../PipelineTemplateDescriptionContainer.java | 4 +-
.../model/template/PipelineTemplateInvocation.java | 7 +-
.../org/apache/streampipes/model/util/Cloner.java | 46 -
.../streampipes/model/util/ElementIdGenerator.java | 2 +-
.../apache/streampipes/model/util/SchemaUtils.java | 34 -
.../matching/output/ListOutputSchemaGenerator.java | 1 -
.../matching/output/PropertyDuplicateRemover.java | 3 -
.../output/TransformOutputSchemaGenerator.java | 7 +-
.../manager/matching/v2/StreamMatch.java | 29 +-
.../manager/matching/v2/StreamQualityMatch.java | 45 -
.../monitoring/pipeline/ExtensionsLogProvider.java | 6 +-
.../runtime/HeartbeatMessageGenerator.java | 38 -
.../runtime/RuntimeVerificationResult.java | 23 -
.../verification/runtime/RuntimeVerifier.java | 23 -
.../verification/runtime/SourceSchemaVerifier.java | 41 -
.../verification/structure/GeneralSecVerifier.java | 23 -
.../verification/structure/GeneralSepVerifier.java | 23 -
.../structure/GeneralSepaVerifier.java | 23 -
.../structure/StreamRequirementVerifier.java | 23 -
.../verification/structure/StreamVerifier.java | 41 -
.../AbstractPipelineElementResourceManager.java | 3 +-
.../streampipes/sdk/builder/DataSourceBuilder.java | 50 -
.../sdk/builder/PrimitivePropertyBuilder.java | 21 -
.../storage/couchdb/serializer/GsonSerializer.java | 8 -
ui/.eslintignore | 1 -
ui/.prettierignore | 1 -
.../fixtures/connect/format/csvWithComma.csv | 2 +
.../fixtures/connect/format/csvWithHeader.csv | 2 +
.../fixtures/connect/format/csvWithoutHeader.csv | 1 +
ui/cypress/fixtures/connect/format/geoJson.json | 16 +
ui/cypress/fixtures/connect/format/jsonArray.json | 9 +
.../fixtures/connect/format/jsonArrayField.json | 11 +
ui/cypress/fixtures/connect/format/jsonObject.json | 7 +
ui/cypress/fixtures/connect/format/xmlObject.xml | 28 +
ui/cypress/support/utils/connect/ConnectBtns.ts | 43 +
ui/cypress/support/utils/connect/ConnectUtils.ts | 12 +-
ui/cypress/tests/adapter/formats/format.spec.ts | 221 +++
ui/package-lock.json | 14 +-
ui/package.json | 2 +-
.../src/lib/model/gen/streampipes-model.ts | 1403 ++++++--------------
.../breadcrumb/breadcrumb.component.html | 35 +-
.../breadcrumb/breadcrumb.component.scss | 20 +-
.../components/breadcrumb/breadcrumb.component.ts | 44 +-
.../core/components/iconbar/iconbar.component.html | 82 +-
.../core/components/iconbar/iconbar.component.scss | 4 +-
.../core/components/iconbar/iconbar.component.ts | 40 +-
.../streampipes/streampipes.component.html | 10 +-
.../streampipes/streampipes.component.scss | 10 +-
.../streampipes/streampipes.component.ts | 21 +-
.../core/components/toolbar/toolbar.component.html | 204 ++-
.../core/components/toolbar/toolbar.component.scss | 61 +-
.../core/components/toolbar/toolbar.component.ts | 170 +--
ui/src/app/core/core.module.ts | 57 +-
ui/src/app/editor/editor.component.ts | 20 +-
126 files changed, 1622 insertions(+), 3491 deletions(-)
delete mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/EventStreamDeclarer.java
delete mode 100644 streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/declarer/SemanticEventProducerDeclarer.java
copy streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/{guess/SchemaGuesser.java => format/geojson/GeoJsonConstants.java} (56%)
copy streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/{object/JsonObjectParser.java => AbstractJsonParser.java} (67%)
copy streampipes-model/src/main/java/org/apache/streampipes/model/util/SchemaUtils.java => streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/EventSchemaUtils.java (74%)
copy streampipes-data-explorer/src/test/java/org/apache/streampipesdataexplorer/v4/query/writer/item/TestCsvItemWriter.java => streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/xml/XmlMapConverterTest.java (52%)
rename streampipes-model/src/test/java/org/apache/streampipes/model/util/SchemaUtilsTest.java => streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/util/EventSchemaUtilsTest.java (81%)
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/ApplicationLink.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/base/AbstractStreamPipesEntity.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/base/Namespaces.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/base/UnnamedStreamPipesEntity.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSourceDescription.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/Accuracy.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/EventPropertyQualityDefinition.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/EventPropertyQualityRequirement.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/EventStreamQualityDefinition.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/EventStreamQualityRequirement.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/Frequency.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/Latency.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/MeasurementCapability.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/MeasurementObject.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/MeasurementProperty.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/MeasurementRange.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/Precision.java
delete mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/quality/Resolution.java
delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/StreamQualityMatch.java
delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/runtime/HeartbeatMessageGenerator.java
delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/runtime/RuntimeVerificationResult.java
delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/runtime/RuntimeVerifier.java
delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/runtime/SourceSchemaVerifier.java
delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/structure/GeneralSecVerifier.java
delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/structure/GeneralSepVerifier.java
delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/structure/GeneralSepaVerifier.java
delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/structure/StreamRequirementVerifier.java
delete mode 100644 streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/structure/StreamVerifier.java
delete mode 100644 streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/DataSourceBuilder.java
create mode 100644 ui/cypress/fixtures/connect/format/csvWithComma.csv
create mode 100644 ui/cypress/fixtures/connect/format/csvWithHeader.csv
create mode 100644 ui/cypress/fixtures/connect/format/csvWithoutHeader.csv
create mode 100644 ui/cypress/fixtures/connect/format/geoJson.json
create mode 100644 ui/cypress/fixtures/connect/format/jsonArray.json
create mode 100644 ui/cypress/fixtures/connect/format/jsonArrayField.json
create mode 100644 ui/cypress/fixtures/connect/format/jsonObject.json
create mode 100644 ui/cypress/fixtures/connect/format/xmlObject.xml
create mode 100644 ui/cypress/tests/adapter/formats/format.spec.ts
[streampipes] 02/02: [#1085] Extract find timestamp property from FileStreamProtocol to EventSchemaUtils
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch SP-1085
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 02059a87d314191368147d2d4d38d19d70abb599
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Fri Jan 13 14:12:23 2023 +0100
[#1085] Extract find timestamp property from FileStreamProtocol to EventSchemaUtils
---
streampipes-extensions-management/pom.xml | 6 ++++
.../connect/adapter/AdapterPipelineGenerator.java | 9 ++++--
.../adapter/model/generic/GenericAdapter.java | 4 ---
.../pipeline/AdapterEventPreviewPipeline.java | 6 ++++
.../adapter/model/pipeline/AdapterPipeline.java | 8 +++--
.../management/util/EventSchemaUtils.java | 29 ++----------------
.../management/util/EventSchemaUtilsTest.java | 18 ++++--------
.../iiot/protocol/stream/FileStreamProtocol.java | 34 ++++------------------
streampipes-model/pom.xml | 5 ----
.../apache/streampipes/model/util/SchemaUtils.java | 34 ----------------------
10 files changed, 36 insertions(+), 117 deletions(-)
diff --git a/streampipes-extensions-management/pom.xml b/streampipes-extensions-management/pom.xml
index 883d30fb4..0b6c9e15d 100644
--- a/streampipes-extensions-management/pom.xml
+++ b/streampipes-extensions-management/pom.xml
@@ -96,6 +96,12 @@
<artifactId>streampipes-service-discovery</artifactId>
<version>0.91.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-test-utils</artifactId>
+ <version>0.91.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.streampipes</groupId>
<artifactId>streampipes-vocabulary</artifactId>
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java
index c344466bc..32a913f99 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java
@@ -74,12 +74,15 @@ public class AdapterPipelineGenerator {
if (adapterDescription.getEventGrounding() != null
&& adapterDescription.getEventGrounding().getTransportProtocol() != null
&& adapterDescription.getEventGrounding().getTransportProtocol().getBrokerHostname() != null) {
- return new AdapterPipeline(pipelineElements, getAdapterSink(adapterDescription));
+ return new AdapterPipeline(
+ pipelineElements,
+ getAdapterSink(adapterDescription),
+ adapterDescription.getEventSchema());
}
DebugSinkRuleDescription debugSinkRuleDescription = getDebugRule(adapterDescription.getRules());
if (debugSinkRuleDescription != null) {
- return new AdapterPipeline(pipelineElements, new DebugAdapterSink());
+ return new AdapterPipeline(pipelineElements, new DebugAdapterSink(), adapterDescription.getEventSchema());
}
return new AdapterPipeline(pipelineElements, adapterDescription.getEventSchema());
@@ -176,7 +179,7 @@ public class AdapterPipelineGenerator {
}
private boolean isPrioritized(SpProtocol prioritizedProtocol,
- Class<?> protocolClass) {
+ Class<?> protocolClass) {
return prioritizedProtocol.getProtocolClass().equals(protocolClass.getCanonicalName());
}
}
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/generic/GenericAdapter.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/generic/GenericAdapter.java
index 54f16fac4..fa8f864f0 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/generic/GenericAdapter.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/generic/GenericAdapter.java
@@ -67,10 +67,6 @@ public abstract class GenericAdapter<T extends AdapterDescription> extends Adapt
IProtocol protocolInstance = this.protocol.getInstance(protocolDescription, parser, format);
this.protocol = protocolInstance;
- //TODO remove
-// EventSchema eventSchema = adapterDescription.getEventSchema();
-// this.protocol.setEventSchema(eventSchema);
-
logger.debug("Start adatper with format: " + format.getId() + " and " + protocol.getId());
protocolInstance.run(adapterPipeline);
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/pipeline/AdapterEventPreviewPipeline.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/pipeline/AdapterEventPreviewPipeline.java
index 3367d2155..17a70bfcd 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/pipeline/AdapterEventPreviewPipeline.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/pipeline/AdapterEventPreviewPipeline.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement;
import org.apache.streampipes.extensions.management.connect.adapter.AdapterPipelineGenerator;
import org.apache.streampipes.model.connect.guess.AdapterEventPreview;
import org.apache.streampipes.model.connect.guess.GuessTypeInfo;
+import org.apache.streampipes.model.schema.EventSchema;
import java.util.List;
import java.util.Map;
@@ -79,4 +80,9 @@ public class AdapterEventPreviewPipeline implements IAdapterPipeline {
.collect(Collectors.toMap(Map.Entry::getKey,
e -> new GuessTypeInfo(e.getValue().getClass().getCanonicalName(), e.getValue())));
}
+
+ @Override
+ public EventSchema getResultingEventSchema() {
+ return null;
+ }
}
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/pipeline/AdapterPipeline.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/pipeline/AdapterPipeline.java
index d1a0bc1c6..7edd585ec 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/pipeline/AdapterPipeline.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/model/pipeline/AdapterPipeline.java
@@ -32,14 +32,18 @@ public class AdapterPipeline implements IAdapterPipeline {
private EventSchema resultingEventSchema;
- public AdapterPipeline(List<IAdapterPipelineElement> pipelineElements, EventSchema resultingEventSchema) {
+ public AdapterPipeline(List<IAdapterPipelineElement> pipelineElements,
+ EventSchema resultingEventSchema) {
this.pipelineElements = pipelineElements;
this.resultingEventSchema = resultingEventSchema;
}
- public AdapterPipeline(List<IAdapterPipelineElement> pipelineElements, IAdapterPipelineElement pipelineSink) {
+ public AdapterPipeline(List<IAdapterPipelineElement> pipelineElements,
+ IAdapterPipelineElement pipelineSink,
+ EventSchema resultingEventSchema) {
this.pipelineElements = pipelineElements;
this.pipelineSink = pipelineSink;
+ this.resultingEventSchema = resultingEventSchema;
}
@Override
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/util/SchemaUtils.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/EventSchemaUtils.java
similarity index 74%
copy from streampipes-model/src/main/java/org/apache/streampipes/model/util/SchemaUtils.java
copy to streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/EventSchemaUtils.java
index ba8a15587..f0a702044 100644
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/util/SchemaUtils.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/EventSchemaUtils.java
@@ -16,7 +16,7 @@
*
*/
-package org.apache.streampipes.model.util;
+package org.apache.streampipes.extensions.management.util;
import org.apache.streampipes.model.schema.EventProperty;
import org.apache.streampipes.model.schema.EventPropertyNested;
@@ -25,35 +25,10 @@ import org.apache.streampipes.model.schema.EventSchema;
import org.apache.streampipes.vocabulary.SO;
import java.net.URI;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
-public class SchemaUtils {
-
- public static Map<String, Object> toRuntimeMap(List<EventProperty> eps) {
- return toUntypedRuntimeMap(eps);
- }
-
- public static Map<String, Object> toUntypedRuntimeMap(List<EventProperty> eps) {
- Map<String, Object> propertyMap = new HashMap<>();
-
- for (EventProperty p : eps) {
- propertyMap.putAll(PropertyUtils.getUntypedRuntimeFormat(p));
- }
- return propertyMap;
- }
-
- public static List<String> toPropertyList(List<EventProperty> eps) {
- List<String> properties = new ArrayList<>();
-
- for (EventProperty p : eps) {
- properties.addAll(PropertyUtils.getFullPropertyName(p, ""));
- }
- return properties;
- }
+public class EventSchemaUtils {
/**
* Returns the timestamp property of an event schema as an {@code Optional}.
diff --git a/streampipes-model/src/test/java/org/apache/streampipes/model/util/SchemaUtilsTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/util/EventSchemaUtilsTest.java
similarity index 81%
rename from streampipes-model/src/test/java/org/apache/streampipes/model/util/SchemaUtilsTest.java
rename to streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/util/EventSchemaUtilsTest.java
index 4dd3bb5bc..58797f97f 100644
--- a/streampipes-model/src/test/java/org/apache/streampipes/model/util/SchemaUtilsTest.java
+++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/util/EventSchemaUtilsTest.java
@@ -16,7 +16,7 @@
*
*/
-package org.apache.streampipes.model.util;
+package org.apache.streampipes.extensions.management.util;
import org.apache.streampipes.model.schema.EventProperty;
import org.apache.streampipes.test.generator.EventPropertyNestedTestBuilder;
@@ -30,8 +30,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-
-public class SchemaUtilsTest {
+public class EventSchemaUtilsTest {
EventProperty timestampProperty = EventPropertyPrimitiveTestBuilder.create()
.withSemanticType(SO.DATE_TIME)
@@ -46,7 +45,7 @@ public class SchemaUtilsTest {
EventPropertyPrimitiveTestBuilder.create().build())
.build();
- var result = SchemaUtils.getTimestampProperty(eventSchema);
+ var result = EventSchemaUtils.getTimestampProperty(eventSchema);
assertFalse(result.isPresent());
}
@@ -57,12 +56,7 @@ public class SchemaUtilsTest {
.withEventProperty(timestampProperty)
.build();
-// var timestampProperty = new EventPropertyPrimitive();
-// timestampProperty.setDomainProperties(List.of(URI.create(SO.DATE_TIME)));
-// timestampProperty.setRuntimeName("timestamp");
-// eventSchema.addEventProperty(timestampProperty);
-
- var result = SchemaUtils.getTimestampProperty(eventSchema);
+ var result = EventSchemaUtils.getTimestampProperty(eventSchema);
assertTrue(result.isPresent());
assertEquals(result.get(), timestampProperty);
@@ -78,11 +72,9 @@ public class SchemaUtilsTest {
.build())
.build();
- var result = SchemaUtils.getTimestampProperty(eventSchema);
+ var result = EventSchemaUtils.getTimestampProperty(eventSchema);
assertTrue(result.isPresent());
assertEquals(result.get(), timestampProperty);
}
-
-
}
\ No newline at end of file
diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileStreamProtocol.java b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileStreamProtocol.java
index 974265bc0..a07813dbb 100644
--- a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileStreamProtocol.java
+++ b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileStreamProtocol.java
@@ -30,13 +30,10 @@ import org.apache.streampipes.extensions.management.connect.adapter.preprocessin
import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.SendToJmsAdapterSink;
import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.SendToKafkaAdapterSink;
import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.SendToMqttAdapterSink;
+import org.apache.streampipes.extensions.management.util.EventSchemaUtils;
import org.apache.streampipes.model.AdapterType;
import org.apache.streampipes.model.connect.grounding.ProtocolDescription;
import org.apache.streampipes.model.connect.guess.GuessSchema;
-import org.apache.streampipes.model.schema.EventProperty;
-import org.apache.streampipes.model.schema.EventPropertyList;
-import org.apache.streampipes.model.schema.EventPropertyNested;
-import org.apache.streampipes.model.schema.EventPropertyPrimitive;
import org.apache.streampipes.model.schema.EventSchema;
import org.apache.streampipes.sdk.builder.adapter.ProtocolDescriptionBuilder;
import org.apache.streampipes.sdk.extractor.StaticPropertyExtractor;
@@ -53,7 +50,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
public class FileStreamProtocol extends Protocol {
@@ -85,7 +81,7 @@ public class FileStreamProtocol extends Protocol {
@Override
public void run(IAdapterPipeline adapterPipeline) {
- String timestampKey = getTimestampKey(adapterPipeline.getResultingEventSchema().getEventProperties(), "");
+ String timestampKey = getTimestampKey(adapterPipeline.getResultingEventSchema());
// exchange adapter pipeline sink with special purpose replay sink for file replay
if (adapterPipeline.getPipelineSink() instanceof SendToKafkaAdapterSink) {
@@ -172,29 +168,9 @@ public class FileStreamProtocol extends Protocol {
return new FileStreamProtocol(parser, format, fileName, replaceTimestamp, speedUp, timeBetweenReplay);
}
- private String getTimestampKey(List<EventProperty> eventProperties, String prefixKey) {
- String result = null;
- for (EventProperty eventProperty : eventProperties) {
- if (eventProperty instanceof EventPropertyPrimitive && eventProperty.getDomainProperties() != null) {
- for (int i = eventProperty.getDomainProperties().size() - 1; i >= 0; i--) {
- if (eventProperty.getDomainProperties().get(0).toString().equals("http://schema.org/DateTime")) {
- result = prefixKey + eventProperty.getRuntimeName();
- }
- }
- } else if (eventProperty instanceof EventPropertyNested
- && ((EventPropertyNested) eventProperty).getEventProperties() != null) {
- result = getTimestampKey(((EventPropertyNested) eventProperty).getEventProperties(),
- prefixKey + eventProperty.getRuntimeName() + ".");
- } else if (eventProperty instanceof EventPropertyList
- && ((EventPropertyList) eventProperty).getEventProperty() != null) {
- result = getTimestampKey(Arrays.asList(((EventPropertyList) eventProperty).getEventProperty()),
- prefixKey + eventProperty.getRuntimeName() + ".");
- }
- if (result != null) {
- return result;
- }
- }
- return result;
+ private String getTimestampKey(EventSchema eventSchema) {
+ var timestampProperty = EventSchemaUtils.getTimestampProperty(eventSchema);
+ return timestampProperty.get().getRuntimeName();
}
@Override
diff --git a/streampipes-model/pom.xml b/streampipes-model/pom.xml
index 994507656..40bb1ca79 100644
--- a/streampipes-model/pom.xml
+++ b/streampipes-model/pom.xml
@@ -38,11 +38,6 @@
<artifactId>streampipes-model-shared</artifactId>
<version>0.91.0-SNAPSHOT</version>
</dependency>
- <dependency>
- <groupId>org.apache.streampipes</groupId>
- <artifactId>streampipes-test-utils</artifactId>
- <version>0.91.0-SNAPSHOT</version>
- </dependency>
<dependency>
<groupId>org.apache.streampipes</groupId>
<artifactId>streampipes-logging</artifactId>
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/util/SchemaUtils.java b/streampipes-model/src/main/java/org/apache/streampipes/model/util/SchemaUtils.java
index ba8a15587..48738c143 100644
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/util/SchemaUtils.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/util/SchemaUtils.java
@@ -19,17 +19,11 @@
package org.apache.streampipes.model.util;
import org.apache.streampipes.model.schema.EventProperty;
-import org.apache.streampipes.model.schema.EventPropertyNested;
-import org.apache.streampipes.model.schema.EventPropertyPrimitive;
-import org.apache.streampipes.model.schema.EventSchema;
-import org.apache.streampipes.vocabulary.SO;
-import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
public class SchemaUtils {
@@ -55,32 +49,4 @@ public class SchemaUtils {
return properties;
}
- /**
- * Returns the timestamp property of an event schema as an {@code Optional}.
- *
- * <p> The method checks all properties if they are of type {@code EventPropertyPrimitive} and if their domain
- * properties contains the uri http://schema.org/DateTime </p>
- *
- * @param eventSchema the event schema for which the timestamp property is to be returned
- * @return an {@code Optional} containing the timestamp property, or an empty {@code Optional} if
- * no such property was found
- */
- public static Optional<EventPropertyPrimitive> getTimestampProperty(EventSchema eventSchema) {
- return getTimstampProperty(eventSchema.getEventProperties());
- }
-
-
- private static Optional<EventPropertyPrimitive> getTimstampProperty(List<EventProperty> eventProperties) {
- for (EventProperty ep : eventProperties) {
- if (ep instanceof EventPropertyPrimitive && ep.getDomainProperties().contains(URI.create(SO.DATE_TIME))) {
- return Optional.of((EventPropertyPrimitive) ep);
- }
-
- if (ep instanceof EventPropertyNested) {
- return getTimstampProperty(((EventPropertyNested) ep).getEventProperties());
- }
- }
-
- return Optional.empty();
- }
}
[streampipes] 01/02: Merge remote-tracking branch 'origin/SP-1065' into SP-1085
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch SP-1085
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 77354bffcc8e59956c142b566b8935065d26a955
Merge: d9250f91f 8d07cafd2
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Fri Jan 13 10:50:51 2023 +0100
Merge remote-tracking branch 'origin/SP-1065' into SP-1085
pom.xml | 12 +-
.../api/declarer/EventStreamDeclarer.java | 25 -
.../declarer/SemanticEventProducerDeclarer.java | 28 -
streampipes-extensions-management/pom.xml | 14 +-
.../connect/adapter/format/csv/CsvParser.java | 4 +-
.../adapter/format/geojson/GeoJsonConstants.java | 14 +-
.../adapter/format/geojson/GeoJsonFormat.java | 28 +-
.../adapter/format/geojson/GeoJsonParser.java | 99 +-
...onObjectParser.java => AbstractJsonParser.java} | 49 +-
.../adapter/format/json/arraykey/JsonParser.java | 32 +-
.../format/json/arraynokey/JsonArrayParser.java | 36 +-
.../format/json/object/JsonObjectParser.java | 47 +-
.../adapter/format/xml/XmlMapConverter.java | 20 +-
.../connect/adapter/format/xml/XmlParser.java | 37 +-
.../adapter/format/xml/XmlMapConverterTest.java | 56 +
.../streampipes-connect-adapters-iiot/pom.xml | 6 +
.../apache/streampipes/model/ApplicationLink.java | 95 --
.../org/apache/streampipes/model/SpDataSet.java | 11 +-
.../org/apache/streampipes/model/SpDataStream.java | 67 -
.../streampipes/model/SpDataStreamContainer.java | 8 +-
.../model/base/AbstractStreamPipesEntity.java | 71 -
.../model/base/NamedStreamPipesEntity.java | 50 +-
.../model/base/UnnamedStreamPipesEntity.java | 45 -
.../model/connect/guess/GuessSchema.java | 7 +-
.../rules/TransformationRuleDescription.java | 9 +-
.../model/dashboard/DashboardWidgetSettings.java | 3 +-
.../model/datalake/DataLakeMeasure.java | 16 +-
.../model/graph/DataSourceDescription.java | 93 --
.../model/grounding/EventGrounding.java | 4 +-
.../model/grounding/TopicDefinition.java | 7 +-
.../model/grounding/TransportFormat.java | 5 +-
.../model/grounding/TransportProtocol.java | 18 +-
.../model/grounding/WildcardTopicMapping.java | 4 +-
.../monitoring/ElementStatusInfoSettings.java | 5 +-
.../streampipes/model/output/OutputStrategy.java | 6 +-
.../model/output/PropertyRenameRule.java | 5 +-
.../model/output/TransformOperation.java | 5 +-
.../apache/streampipes/model/quality/Accuracy.java | 47 -
.../quality/EventPropertyQualityDefinition.java | 41 -
.../quality/EventPropertyQualityRequirement.java | 82 --
.../quality/EventStreamQualityDefinition.java | 40 -
.../quality/EventStreamQualityRequirement.java | 64 -
.../streampipes/model/quality/Frequency.java | 52 -
.../apache/streampipes/model/quality/Latency.java | 63 -
.../model/quality/MeasurementCapability.java | 54 -
.../model/quality/MeasurementObject.java | 54 -
.../model/quality/MeasurementProperty.java | 41 -
.../model/quality/MeasurementRange.java | 61 -
.../streampipes/model/quality/Precision.java | 48 -
.../streampipes/model/quality/Resolution.java | 48 -
.../model/runtime/RuntimeOptionsRequest.java | 6 +-
.../streampipes/model/schema/Enumeration.java | 1 -
.../streampipes/model/schema/EventProperty.java | 69 +-
.../model/schema/EventPropertyPrimitive.java | 10 -
.../streampipes/model/schema/EventSchema.java | 4 +-
.../model/schema/QuantitativeValue.java | 3 -
.../model/schema/ValueSpecification.java | 10 +-
.../streampipes/model/staticproperty/Option.java | 26 +-
.../staticproperty/PropertyValueSpecification.java | 4 +-
.../model/staticproperty/StaticProperty.java | 6 +-
.../staticproperty/StaticPropertyAlternative.java | 13 +
.../model/staticproperty/SupportedProperty.java | 4 +-
.../model/template/BoundPipelineElement.java | 5 +-
.../PipelineTemplateDescriptionContainer.java | 4 +-
.../model/template/PipelineTemplateInvocation.java | 7 +-
.../org/apache/streampipes/model/util/Cloner.java | 46 -
.../streampipes/model/util/ElementIdGenerator.java | 2 +-
.../matching/output/ListOutputSchemaGenerator.java | 1 -
.../matching/output/PropertyDuplicateRemover.java | 3 -
.../output/TransformOutputSchemaGenerator.java | 7 +-
.../manager/matching/v2/StreamMatch.java | 29 +-
.../manager/matching/v2/StreamQualityMatch.java | 45 -
.../monitoring/pipeline/ExtensionsLogProvider.java | 6 +-
.../runtime/HeartbeatMessageGenerator.java | 38 -
.../runtime/RuntimeVerificationResult.java | 23 -
.../verification/runtime/RuntimeVerifier.java | 23 -
.../verification/runtime/SourceSchemaVerifier.java | 41 -
.../verification/structure/GeneralSecVerifier.java | 23 -
.../verification/structure/GeneralSepVerifier.java | 23 -
.../structure/GeneralSepaVerifier.java | 23 -
.../structure/StreamRequirementVerifier.java | 23 -
.../verification/structure/StreamVerifier.java | 41 -
.../AbstractPipelineElementResourceManager.java | 3 +-
.../streampipes/sdk/builder/DataSourceBuilder.java | 50 -
.../sdk/builder/PrimitivePropertyBuilder.java | 21 -
.../storage/couchdb/serializer/GsonSerializer.java | 8 -
ui/.eslintignore | 1 -
ui/.prettierignore | 1 -
.../fixtures/connect/format/csvWithComma.csv | 2 +
.../fixtures/connect/format/csvWithHeader.csv | 2 +
.../fixtures/connect/format/csvWithoutHeader.csv | 1 +
ui/cypress/fixtures/connect/format/geoJson.json | 16 +
ui/cypress/fixtures/connect/format/jsonArray.json | 9 +
.../fixtures/connect/format/jsonArrayField.json | 11 +
ui/cypress/fixtures/connect/format/jsonObject.json | 7 +
ui/cypress/fixtures/connect/format/xmlObject.xml | 28 +
ui/cypress/support/utils/connect/ConnectBtns.ts | 43 +
ui/cypress/support/utils/connect/ConnectUtils.ts | 12 +-
ui/cypress/tests/adapter/formats/format.spec.ts | 221 +++
ui/package-lock.json | 14 +-
ui/package.json | 2 +-
.../src/lib/model/gen/streampipes-model.ts | 1403 ++++++--------------
.../breadcrumb/breadcrumb.component.html | 35 +-
.../breadcrumb/breadcrumb.component.scss | 20 +-
.../components/breadcrumb/breadcrumb.component.ts | 44 +-
.../core/components/iconbar/iconbar.component.html | 82 +-
.../core/components/iconbar/iconbar.component.scss | 4 +-
.../core/components/iconbar/iconbar.component.ts | 40 +-
.../streampipes/streampipes.component.html | 10 +-
.../streampipes/streampipes.component.scss | 10 +-
.../streampipes/streampipes.component.ts | 21 +-
.../core/components/toolbar/toolbar.component.html | 204 ++-
.../core/components/toolbar/toolbar.component.scss | 61 +-
.../core/components/toolbar/toolbar.component.ts | 170 +--
ui/src/app/core/core.module.ts | 57 +-
ui/src/app/editor/editor.component.ts | 20 +-
116 files changed, 1620 insertions(+), 3318 deletions(-)