You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by wi...@apache.org on 2020/11/13 15:23:47 UTC
[incubator-streampipes] 01/01: Merge branch 'dev' into
STREAMPIPES-252
This is an automated email from the ASF dual-hosted git repository.
wiener pushed a commit to branch STREAMPIPES-252
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 8f1e08bcb66a3e3e8c7fb0eec6c04ac60a4e7135
Merge: 4d3cdcd b0b157a
Author: Patrick Wiener <wi...@fzi.de>
AuthorDate: Fri Nov 13 16:23:00 2020 +0100
Merge branch 'dev' into STREAMPIPES-252
.github/workflows/build.yml | 2 +-
.gitlab-ci.yml | 2 +-
.../all_pipeline_elements_jvm.xml | 1 +
.idea/runConfigurations/sinks_internal_jvm.xml | 2 +-
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../main/resources/archetype-resources/Dockerfile | 2 +-
docker-compose.yml | 4 +-
streampipes-backend/Dockerfile | 2 +-
.../backend/StreamPipesResourceConfig.java | 2 +
streampipes-backend/src/main/resources/shiro.ini | 7 +
streampipes-connect-container-master/Dockerfile | 2 +-
.../streampipes/connect/adapter/Adapter.java | 13 +-
.../elements/SendToBrokerAdapterSink.java | 4 +-
...ink.java => SendToBrokerReplayAdapterSink.java} | 22 +-
.../elements/SendToKafkaAdapterSink.java | 1 -
.../TransformValueAdapterPipelineElement.java | 7 +-
.../value/CorrectionValueTransformationRule.java | 93 ++++
.../transform/value/ValueEventTransformer.java | 38 +-
.../messaging/jms/ActiveMQPublisher.java | 22 +-
.../apache/streampipes/model/client/Category.java | 3 +-
.../rules/TransformationRuleDescription.java | 10 +-
...rrectionValueTransformationRuleDescription.java | 80 +++
.../value/ValueTransformationRuleDescription.java | 1 +
.../streampipes/model/labeling/Category.java | 67 +++
.../apache/streampipes/model/labeling/Label.java | 73 +++
.../manager/setup/CouchDbInstallationStep.java | 24 +
.../org/apache/streampipes/rest/api/ICategory.java | 28 +-
.../org/apache/streampipes/rest/api/ILabel.java | 31 +-
.../streampipes/rest/impl/CategoryResource.java | 116 ++++
.../streampipes/rest/impl/LabelResource.java | 144 +++++
.../rest/impl/datalake/DataLakeManagementV3.java | 4 +-
.../rest/impl/datalake/DataLakeResourceV3.java | 9 +-
.../serializers/json/GsonSerializer.java | 8 +-
.../jsonld/CustomAnnotationProvider.java | 7 +-
.../streampipes/storage/api/ICategoryStorage.java | 27 +-
.../streampipes/storage/api/ILabelStorage.java | 31 +-
.../streampipes/storage/api/INoSqlStorage.java | 5 +
.../storage/couchdb/CouchDbStorageManager.java | 6 +
.../storage/couchdb/impl/CategoryStorageImpl.java | 57 ++
.../storage/couchdb/impl/LabelStorageImpl.java | 76 +++
.../streampipes/storage/couchdb/utils/Utils.java | 12 +
.../apache/streampipes/vocabulary/StreamPipes.java | 4 +-
ui/package.json | 34 +-
.../event-schema/event-schema.component.ts | 3 +-
ui/src/app/connect/connect.module.ts | 4 +-
.../edit-correction-value.component.css} | 4 +-
.../edit-correction-value.component.html | 39 ++
.../edit-correction-value.component.ts | 55 ++
.../edit-event-property-primitive.component.html | 1 +
.../edit-event-property-primitive.component.ts | 9 +-
.../edit-event-property.component.html | 11 +-
.../edit-event-property.component.ts | 29 +-
.../services/transformation-rule.service.ts | 41 ++
ui/src/app/core-model/coco/Annotation.ts | 2 +
ui/src/app/core-model/coco/Category.ts | 10 +-
ui/src/app/core-model/gen/streampipes-model.ts | 99 +++-
.../datalake/datalake-rest.service.ts | 6 +-
.../semantic-type/semantic-type-utils.service.ts | 5 +
ui/src/app/core-ui/core-ui.module.ts | 118 ++--
.../image-annotations.component.html | 16 +-
.../image-annotations.component.ts | 21 +-
.../image-labels/image-labels.component.html | 39 --
.../image-labels/image-labels.component.ts | 73 ---
.../image-categorize.component.html | 5 +-
.../image-categorize/image-categorize.component.ts | 4 +-
.../image-labeling/image-labeling.component.html | 5 +-
.../image-labeling/image-labeling.component.ts | 45 +-
.../image/services/BrushLabeling.service.ts | 5 +-
.../core-ui/image/services/CocoFormat.service.ts | 22 +-
.../image/services/PolygonLabeling.service.ts | 7 +-
.../image/services/ReactLabeling.service.ts | 9 +-
.../configure-labels.component.css} | 4 +-
.../configure-labels.component.html | 66 +++
.../configure-labels/configure-labels.component.ts | 120 +++++
.../label-list-item/label-list-item.component.css} | 4 +-
.../label-list-item.component.html} | 27 +-
.../label-list-item/label-list-item.component.ts | 59 ++
.../select-label/select-label.component.css} | 0
.../select-label/select-label.component.html | 43 ++
.../select-label/select-label.component.ts | 81 +++
.../app/core-ui/labels/services/label.service.ts | 94 ++++
.../label-selection/label-selection.component.html | 32 --
.../label-selection/label-selection.component.ts | 61 ---
.../change-chartmode/change-chartmode.dialog.css | 29 -
.../change-chartmode/change-chartmode.dialog.ts | 35 --
.../dialogs/labeling/labeling.dialog.css | 36 --
.../dialogs/labeling/labeling.dialog.html | 36 --
.../dialogs/labeling/labeling.dialog.ts | 52 --
.../labeling-tool/labeling-tool.module.ts | 90 ----
...data-explorer-dashboard-overview.component.html | 7 +-
.../data-explorer-dashboard-overview.component.ts | 7 +
.../data-explorer-dashboard-widget.component.html | 2 +-
.../data-explorer-dashboard-widget.component.ts | 2 +
.../widgets/base/base-data-explorer-widget.ts | 14 +-
.../point-selection-info.component.css | 0
.../point-selection-info.component.html | 0
.../point-selection-info.component.ts | 2 +-
.../line-chart/line-chart-widget.component.css | 5 +-
.../line-chart/line-chart-widget.component.html | 35 +-
.../line-chart/line-chart-widget.component.ts | 596 ++++++++++-----------
.../widgets/line-chart}/services/color.service.ts | 0
ui/src/app/data-explorer/data-explorer.module.ts | 96 ++--
105 files changed, 2241 insertions(+), 1172 deletions(-)
diff --cc streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/Adapter.java
index 86d15c2,391cb8d..28258ec
--- a/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/Adapter.java
+++ b/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/Adapter.java
@@@ -18,12 -18,9 +18,13 @@@
package org.apache.streampipes.connect.adapter;
+ import org.apache.streampipes.connect.adapter.preprocessing.elements.*;
+ import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription;
+import org.apache.streampipes.config.backend.BackendConfig;
+import org.apache.streampipes.config.backend.SpProtocol;
- import org.apache.streampipes.connect.adapter.preprocessing.elements.*;
+import org.apache.streampipes.model.grounding.JmsTransportProtocol;
import org.apache.streampipes.model.grounding.KafkaTransportProtocol;
+import org.apache.streampipes.model.grounding.MqttTransportProtocol;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.streampipes.connect.adapter.exception.AdapterException;
@@@ -123,10 -105,11 +125,12 @@@ public abstract class Adapter<T extend
AddValueTransformationRuleDescription valueTransformationRuleDescription = getAddValueRule(adapterDescription);
if (valueTransformationRuleDescription != null) {
- pipelineElements.add(new AddValuePipelineElement(valueTransformationRuleDescription.getRuntimeKey(), valueTransformationRuleDescription.getStaticValue()));
+ pipelineElements.add(new AddValuePipelineElement(
+ valueTransformationRuleDescription.getRuntimeKey(),
+ valueTransformationRuleDescription.getStaticValue()));
}
+
// first transform schema before transforming vales
// value rules should use unique keys for of new schema
pipelineElements.add(new TransformSchemaAdapterPipelineElement(adapterDescription.getSchemaRules()));
diff --cc streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/SendToBrokerAdapterSink.java
index 27439fe,0000000..4333111
mode 100644,000000..100644
--- a/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/SendToBrokerAdapterSink.java
+++ b/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/SendToBrokerAdapterSink.java
@@@ -1,105 -1,0 +1,103 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.connect.adapter.preprocessing.elements;
+
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.connect.adapter.model.pipeline.AdapterPipelineElement;
+import org.apache.streampipes.connect.adapter.util.TransportFormatSelector;
+import org.apache.streampipes.dataformat.SpDataFormatDefinition;
+import org.apache.streampipes.messaging.EventProducer;
+import org.apache.streampipes.model.connect.adapter.AdapterDescription;
+import org.apache.streampipes.model.grounding.TransportFormat;
+import org.apache.streampipes.model.grounding.TransportProtocol;
+
++import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public abstract class SendToBrokerAdapterSink<T extends TransportProtocol> implements AdapterPipelineElement {
+
+ protected AdapterDescription adapterDescription;
+ protected SpDataFormatDefinition dataFormatDefinition;
-
+ protected T protocol;
+ private Class<T> protocolClass;
-
+ private EventProducer<T> producer;
+
+ public SendToBrokerAdapterSink(AdapterDescription adapterDescription,
+ Supplier<EventProducer<T>> producerSupplier,
+ Class<T> protocolClass) {
+ this.adapterDescription = adapterDescription;
+ this.producer = producerSupplier.get();
-
+ this.protocol = protocolClass.cast(adapterDescription
+ .getEventGrounding()
+ .getTransportProtocol());
+
+ if ("true".equals(System.getenv("SP_DEBUG"))) {
+ modifyProtocolForDebugging();
+ }
+
+ TransportFormat transportFormat = adapterDescription
+ .getEventGrounding()
+ .getTransportFormats()
+ .get(0);
+
+ this.dataFormatDefinition =
+ new TransportFormatSelector(transportFormat).getDataFormatDefinition();
+
+ try {
+ producer.connect(protocol);
+ } catch (SpRuntimeException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public Map<String, Object> process(Map<String, Object> event) {
+ try {
+ if (event != null) {
+ if ("true".equals(System.getenv("SP_DEBUG_CONNECT"))) {
+ event.put("internal_t2", System.currentTimeMillis());
+ }
+ sendToBroker(dataFormatDefinition.fromMap(event));
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ protected void sendToBroker(byte[] event) throws Exception {
+ producer.publish(event);
+ }
+
+ protected void modifyProtocolForDebugging() {
+
+ }
+
+ public void changeTransportProtocol(T transportProtocol) {
+ try {
+ producer.disconnect();
+ producer.connect(transportProtocol);
+ } catch (SpRuntimeException e) {
+ e.printStackTrace();
+ }
+ }
+
+}
+
+
diff --cc streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/SendToBrokerReplayAdapterSink.java
index 42864fb,42864fb..905b5ee
--- a/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/SendToBrokerReplayAdapterSink.java
+++ b/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/SendToBrokerReplayAdapterSink.java
@@@ -20,23 -20,23 +20,25 @@@ package org.apache.streampipes.connect.
import org.apache.streampipes.connect.adapter.model.pipeline.AdapterPipelineElement;
import org.apache.streampipes.connect.adapter.preprocessing.Util;
--
++import org.apache.streampipes.messaging.EventProducer;
++import org.apache.streampipes.model.grounding.TransportProtocol;
import java.util.List;
import java.util.Map;
++import java.util.function.Supplier;
--public class SendToKafkaReplayAdapterSink implements AdapterPipelineElement {
++public class SendToBrokerReplayAdapterSink implements AdapterPipelineElement {
-- private SendToKafkaAdapterSink sendToKafkaAdapterSink;
++ private final SendToBrokerAdapterSink sendToBrokerAdapterSink;
private long lastEventTimestamp;
-- private List<String> timestampKeys;
-- private boolean replaceTimestamp;
-- private float speedUp;
++ private final List<String> timestampKeys;
++ private final boolean replaceTimestamp;
++ private final float speedUp;
-- public SendToKafkaReplayAdapterSink(SendToKafkaAdapterSink sendToKafkaAdapterSink,
-- String timestampKey, boolean replaceTimestamp, float speedUp) {
-- this.sendToKafkaAdapterSink = sendToKafkaAdapterSink;
++ public SendToBrokerReplayAdapterSink(SendToBrokerAdapterSink sendToBrokerAdapterSink,
++ String timestampKey, boolean replaceTimestamp, float speedUp) {
++ this.sendToBrokerAdapterSink = sendToBrokerAdapterSink;
this.lastEventTimestamp = -1;
this.timestampKeys = Util.toKeyArray(timestampKey);
this.replaceTimestamp = replaceTimestamp;
@@@ -65,7 -65,7 +67,7 @@@
setTimestampInEvent(event, System.currentTimeMillis());
}
}
-- return sendToKafkaAdapterSink.process(event);
++ return sendToBrokerAdapterSink.process(event);
}
private long getTimestampInEvent(Map<String, Object> event) {
diff --cc streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/SendToKafkaAdapterSink.java
index 434dd9c,7670c66..350b124
--- a/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/SendToKafkaAdapterSink.java
+++ b/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/SendToKafkaAdapterSink.java
@@@ -15,10 -15,13 +15,9 @@@
* limitations under the License.
*
*/
-
package org.apache.streampipes.connect.adapter.preprocessing.elements;
- import org.apache.streampipes.config.backend.BackendConfig;
-import org.apache.streampipes.commons.exceptions.SpRuntimeException;
import org.apache.streampipes.connect.adapter.model.pipeline.AdapterPipelineElement;
-import org.apache.streampipes.connect.adapter.util.TransportFormatSelector;
-import org.apache.streampipes.dataformat.SpDataFormatDefinition;
import org.apache.streampipes.messaging.kafka.SpKafkaProducer;
import org.apache.streampipes.model.connect.adapter.AdapterDescription;
import org.apache.streampipes.model.grounding.KafkaTransportProtocol;
diff --cc streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQPublisher.java
index ad7d4b1,db610fc..02df17f
--- a/streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQPublisher.java
+++ b/streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQPublisher.java
@@@ -49,31 -49,19 +49,31 @@@ public class ActiveMQPublisher implemen
}
- // TODO backwards compatibility, remove later
+ @Deprecated
- // public ActiveMQPublisher(String url, String topic) {
- // JmsTransportProtocol protocol = new JmsTransportProtocol();
- // protocol.setBrokerHostname(url.substring(0, url.lastIndexOf(":")));
- // protocol.setPort(Integer.parseInt(url.substring(url.lastIndexOf(":") + 1, url.length())));
- // protocol.setTopicDefinition(new SimpleTopicDefinition(topic));
- // try {
- // connect(protocol);
- // } catch (SpRuntimeException e) {
- // e.printStackTrace();
- // }
- // }
+ public ActiveMQPublisher(String url, String topic) {
+ JmsTransportProtocol protocol = new JmsTransportProtocol();
+ protocol.setBrokerHostname(url.substring(0, url.lastIndexOf(":")));
+ protocol.setPort(Integer.parseInt(url.substring(url.lastIndexOf(":") + 1, url.length())));
+ protocol.setTopicDefinition(new SimpleTopicDefinition(topic));
+ try {
+ connect(protocol);
+ } catch (SpRuntimeException e) {
+ e.printStackTrace();
+ }
+ }
+ public ActiveMQPublisher(String host, int port, String topic) {
+ JmsTransportProtocol protocol = new JmsTransportProtocol();
+ protocol.setBrokerHostname(host);
+ protocol.setPort(port);
+ protocol.setTopicDefinition(new SimpleTopicDefinition(topic));
+ try {
+ connect(protocol);
+ } catch (SpRuntimeException e) {
+ e.printStackTrace();
+ }
+ }
+
public void sendText(String message) throws JMSException {
publish(message.getBytes());
}