You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by wi...@apache.org on 2020/11/20 09:03:02 UTC
[incubator-streampipes] branch edge-extensions updated (505c4c5 ->
3588b8a)
This is an automated email from the ASF dual-hosted git repository.
wiener pushed a change to branch edge-extensions
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git.
from 505c4c5 Merge branch 'dev' into edge-extensions
add 6ddd4ed [STREAMPIPES-249] Add classes for more compact data processor and sink definition
add 36c86b8 Merge branch 'dev' of github.com:apache/incubator-streampipes into dev
add 6cdeb31 [STREAMPIPES-250] Refactor Siddhi wrapper
add 6e1c9ef [STREAMPIPES-249] Add compact data sink
add 8c5beb8 [STREAMPIPES-250] Refactor Siddhi Wrapper
add da995b5 [STREAMPIPES-250] Improve Siddhi wrapper
add a6ac30a Update Jenkinsfile
add 8d79f57 Merge branch 'dev' of github.com:apache/incubator-streampipes into dev
add 8617c35 [STREAMPIPES-250] Add Siddhi EPL object model for select statements
add 16388c1 [hotfix] use correct build-args in CI
add f015efc Merge branch 'dev' of github.com:apache/incubator-streampipes into dev
add 563a3a7 [STREAMPIPES-250] Extend Siddhi object model
add 4b6e64b [hotfix] Fix bug in static mapping property renderer
add f4dcb7d [hotfix] Consider alternatives for mapping property calculation
add 84ed1cc [hotfix] Fix wrong property access in property selector service
add b01ce7c [STREAMPIPES-252] add mqtt transport protocol
add 4d3cdcd [STREAMPIPES-252] add SP_PRIORITIZED_PROTOCOL env variable
add f18f90d [STREAMPIPES-253] add property scope HEADER_PROPERTY
add f97f0bb [hotfix] use adoptopenjdk/openjdk8-openj9:alpine base image
add 9eaea79 [STREAMPIPES-254] add correction value rule to preprocessing pipeline in connect adapters
add a9fe97a Move line chart labeling to data explorer module
add a66efe6 [STREAMPIPES-234] Integrated label component of image labeling
add abc7f8c Merge branch 'dev' into STREAMPIPES-234
add cd7559b Merge branch 'dev' into STREAMPIPES-234
add 3000e99 [STREAMPIPES-234] Labeled time intervals can now be changed
add d3fde33 Merge branch 'dev' into STREAMPIPES-234
add 4332e1e Merge branch 'dev' into STREAMPIPES-234
add edbc416 Move labels component
add 70cc1b1 Create configure labels component
add 6d5a4a4 Barebone impl of Label-API
add e28b4f8 Merge branch 'timeseries-labeling' into STREAMPIPES-234
add d0294a4 Added Category class
add df03e79 Merge pull request #26 from heymarco/STREAMPIPES-234
add e6bf2a4 Add label model to ui
add c80b35c Add color picker to configure label component
add 69cbceb Enable to add and delete labels
add 46cc9c9 Edit categories in label configuration
add 420c1c3 Add delete functionality for label categories
add 0ae7c55 Labeling api WIP
add 3207736 Adapted routing for labeling api and changed category class
add 820b546 Merge pull request #27 from heymarco/STREAMPIPES-234
add 958f50f Store categories and label in db
add c4635da Finish first version that persists labels and categories
add c7fff83 Refactor line chart
add acd61fe Time-series explorer supports now new label colors
add bdd73c3 Merge branch 'dev' into STREAMPIPES-234
add d28ecfd Rework image labeling
add 0cf97fd Image labeling works now with new label model
add 80396a8 Fix image labeling component
add 440ef73 Merge branch 'STREAMPIPES-234' into dev
add d8b77ee Add headers to fix build
add 8a4109a Add missing header to label service in ui
add e19a2cc Merge remote-tracking branch 'refs/remotes/origin/dev' into dev
add a87727b [hotfix] add missing fields
add 72fbcf0 [STREAMPIPES-254] fix ClassCastException from long to double
add 7ded342 [STREAMPIPES-254] additional changes to ClassCastException
add b0b157a [hotfix] use adoptopenjdk/openjdk8-openj9:alpine base image for all Dockerfiles
add 8f1e08b Merge branch 'dev' into STREAMPIPES-252
add 2c4324f [STREAMPIPES-252] add SP_PRIORITIZED_PROTOCOL to .idea run config
add 007bd04 [STREAMPIPES-252] add mqtt transport protocol to flink wrapper
add e994228 [STREAMPIPES-252] remove prioritizedProtocol helper from sdk, fixed typo
add fbaeb08 [STREAMPIPES-252] add mqtt transport protocol to archetypes
add a4940fc [hotfix] temporary commented tess in GroundingServiceTest
add 30b937d [hotfix] update to new syntax to set env files in github actions
add afa6a18 [STREAMPIPES-252] add activemq as default hostname for mqtt
add 1040204 [STREAMPIPES-253] use event schema from adapter description if exists
new fb0521e Merge branch 'dev' into edge-extensions
new f1116ff use updated model
new 3588b8a initial changes to InvocableGraphBuilder
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/build.yml | 38 +-
.gitlab-ci.yml | 2 +-
.../all_pipeline_elements_jvm.xml | 1 +
.idea/runConfigurations/backend.xml | 1 +
.idea/runConfigurations/sinks_internal_jvm.xml | 2 +-
Jenkinsfile | 6 +-
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../archetype-resources/src/main/java/Init.java | 1 +
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 5 +
.../archetype-resources/src/main/java/Init.java | 1 +
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../archetype-resources/src/main/java/Init.java | 1 +
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 5 +
.../archetype-resources/src/main/java/Init.java | 1 +
.../main/resources/archetype-resources/Dockerfile | 2 +-
docker-compose.yml | 4 +-
pom.xml | 1 +
streampipes-backend/Dockerfile | 2 +-
.../backend/StreamPipesResourceConfig.java | 2 +
streampipes-backend/src/main/resources/shiro.ini | 7 +
.../commons/exceptions/SpException.java | 2 +-
.../streampipes/config/backend/BackendConfig.java | 5 +-
.../config/backend/BackendConfigKeys.java | 2 +
.../config/backend/MessagingSettings.java | 33 +-
.../streampipes/config/backend/SpProtocol.java | 10 +-
streampipes-connect-container-master/Dockerfile | 2 +-
.../master/management/WorkerRestClient.java | 2 +-
.../streampipes/connect/adapter/Adapter.java | 60 ++-
.../connect/adapter/GroundingService.java | 36 +-
.../elements/SendToBrokerAdapterSink.java | 4 +-
...ink.java => SendToBrokerReplayAdapterSink.java} | 22 +-
.../elements/SendToJmsAdapterSink.java | 10 +-
.../elements/SendToKafkaAdapterSink.java | 6 +-
.../elements/SendToMqttAdapterSink.java | 14 +-
.../TransformValueAdapterPipelineElement.java | 7 +-
.../value/CorrectionValueTransformationRule.java | 93 ++++
.../transform/value/ValueEventTransformer.java | 38 +-
.../connect/adapter/GroundingServiceTest.java | 128 +++--
.../streampipes/container/util/ConsulUtil.java | 10 +-
.../messaging/jms/ActiveMQPublisher.java | 15 +-
.../streampipes/messaging/jms/ActiveMQUtils.java | 12 +-
.../streampipes/messaging/mqtt/MqttConsumer.java | 13 +-
.../apache/streampipes/model/client/Category.java | 3 +-
.../rules/TransformationRuleDescription.java | 10 +-
...rrectionValueTransformationRuleDescription.java | 80 +++
.../value/ValueTransformationRuleDescription.java | 1 +
.../streampipes/model/labeling/Category.java | 67 +++
.../apache/streampipes/model/labeling/Label.java | 73 +++
.../org/apache/streampipes/model/util/Cloner.java | 9 +-
streampipes-node-controller-container/pom.xml | 3 +-
.../{init => }/NodeControllerContainer.java | 5 +-
.../controller/container/config/ConfigKeys.java | 1 -
.../container/management/info/NodeInfoStorage.java | 2 +-
.../management/resource/ResourceManager.java | 52 +-
.../AbstractNodeContainerResource.java} | 49 +-
.../container/rest/DebugRelayResource.java | 55 ++
.../HealthCheckResource.java} | 11 +-
.../InfoStatusResource.java} | 14 +-
.../NodeControllerResourceConfig.java} | 18 +-
.../PELifeCycleResource.java} | 81 +--
streampipes-pipeline-management/pom.xml | 5 +
.../http/InvocableEntityUrlGenerator.java | 4 +-
.../manager/matching/InvocationGraphBuilder.java | 161 ++++--
.../matching/PipelineVerificationHandler.java | 3 +
.../manager/matching/ProtocolSelector.java | 44 +-
.../runtime/PipelineElementRuntimeInfoFetcher.java | 28 +-
.../manager/setup/CouchDbInstallationStep.java | 24 +
.../org/apache/streampipes/rest/api/ICategory.java | 36 +-
.../org/apache/streampipes/rest/api/ILabel.java | 36 +-
.../streampipes/rest/impl/CategoryResource.java | 116 ++++
.../streampipes/rest/impl/LabelResource.java | 144 +++++
.../rest/impl/datalake/DataLakeManagementV3.java | 4 +-
.../rest/impl/datalake/DataLakeResourceV3.java | 9 +-
.../sdk/extractor/AbstractParameterExtractor.java | 39 ++
.../apache/streampipes/sdk/helpers/Protocols.java | 21 +-
.../sdk/helpers/SupportedProtocols.java | 10 +
.../apache/streampipes/sdk/utils/Datatypes.java | 8 +
.../serializers/json/GsonSerializer.java | 8 +-
.../jsonld/CustomAnnotationProvider.java | 7 +-
.../streampipes/storage/api/ICategoryStorage.java | 35 +-
.../streampipes/storage/api/ILabelStorage.java | 39 +-
.../streampipes/storage/api/INoSqlStorage.java | 5 +
.../storage/couchdb/CouchDbStorageManager.java | 6 +
.../storage/couchdb/impl/CategoryStorageImpl.java | 57 ++
.../storage/couchdb/impl/LabelStorageImpl.java | 76 +++
.../streampipes/storage/couchdb/utils/Utils.java | 12 +
.../apache/streampipes/vocabulary/StreamPipes.java | 5 +-
streampipes-wrapper-distributed/pom.xml | 5 +
.../distributed/runtime/DistributedRuntime.java | 21 +-
.../wrapper/flink/FlinkDataProcessorRuntime.java | 10 +-
.../streampipes/wrapper/flink/FlinkRuntime.java | 22 +-
.../{JmsConsumer.java => JmsFlinkConsumer.java} | 6 +-
.../{JmsConsumer.java => MqttFlinkConsumer.java} | 26 +-
...FlinkJmsProducer.java => JmsFlinkProducer.java} | 4 +-
...linkJmsProducer.java => MqttFlinkProducer.java} | 14 +-
streampipes-wrapper-siddhi/pom.xml | 6 +
.../wrapper/siddhi/constants/SiddhiConstants.java | 55 ++
.../siddhi/constants/SiddhiStreamSelector.java | 17 +-
.../wrapper/siddhi/engine/SiddhiEngine.java | 127 +++++
.../wrapper/siddhi/engine/SiddhiEventEngine.java | 274 +---------
...Callback.java => SiddhiStatementGenerator.java} | 9 +-
.../siddhi/engine/StreamPipesSiddhiProcessor.java | 57 ++
.../engine/{ => callback}/SiddhiDebugCallback.java | 2 +-
.../callback/SiddhiOutputStreamCallback.java | 54 ++
.../SiddhiOutputStreamDebugCallback.java} | 24 +-
.../engine/generator/EventTypeGenerator.java | 77 +++
.../engine/generator/InputStreamNameGenerator.java | 33 +-
.../engine/generator/SiddhiAppGenerator.java | 86 +++
.../generator/SiddhiInvocationConfigGenerator.java | 65 +++
.../wrapper/siddhi/model/EventPropertyDef.java | 30 +-
.../siddhi/model/SiddhiProcessorParams.java | 100 ++++
.../siddhi/query/AbstractQueryGenerator.java | 20 +-
.../wrapper/siddhi/query/FromClause.java | 48 ++
.../wrapper/siddhi/query/SelectClause.java | 78 +++
.../wrapper/siddhi/query/SiddhiStatement.java | 28 +-
.../siddhi/query/expression/EqualsExpression.java | 16 +-
.../siddhi/query/expression/Expression.java | 63 +++
.../siddhi/query/expression/Expressions.java | 157 ++++++
.../query/expression/GreaterEqualsExpression.java | 16 +-
.../query/expression/GreaterThanExpression.java | 16 +-
.../query/expression/LesserEqualsExpression.java | 16 +-
.../query/expression/LesserThanExpression.java | 16 +-
.../query/expression/PropertyExpression.java | 48 ++
.../query/expression/PropertyExpressionBase.java | 6 +-
.../query/expression/PropertyRenameExpression.java | 30 +-
.../query/expression/RelationalOperator.java | 22 +-
.../expression/RelationalOperatorExpression.java | 34 +-
.../query/expression/SequenceExpression.java | 24 +-
.../siddhi/query/expression/SiddhiTimeUnit.java | 18 +-
.../siddhi/query/expression/StreamExpression.java | 58 ++
.../query/expression/StreamFilterExpression.java | 57 ++
.../siddhi/query/expression/WithinExpression.java | 29 +-
.../expression/list/CollectListExpression.java | 31 +-
.../expression/list/ContainsListExpression.java | 52 ++
.../query/expression/math/MathAddExpression.java | 10 +-
.../expression/math/MathDivideExpression.java | 10 +-
.../query/expression/math/MathExpression.java | 44 ++
.../expression/math/MathMultiplyExpression.java | 10 +-
.../siddhi/query/expression/math/MathOperator.java | 23 +-
.../expression/math/MathSubtractExpression.java | 10 +-
.../query/expression/pattern/EveryExpression.java | 26 +-
.../expression/pattern/PatternCountExpression.java | 54 ++
.../expression/pattern/PatternCountOperator.java | 9 +-
.../expression/pattern/PatternExpression.java | 60 +++
.../expression/window/BatchWindowExpression.java | 15 +-
.../expression/window/TimeWindowExpression.java | 29 +-
.../query/expression/window/WindowExpression.java | 28 +-
.../wrapper/siddhi/utils/SiddhiUtils.java | 85 +++
.../wrapper/standalone/ProcessorParams.java | 27 +-
.../streampipes/wrapper/standalone/SinkParams.java | 23 +-
.../standalone/StreamPipesDataProcessor.java | 35 ++
.../wrapper/standalone/StreamPipesDataSink.java | 29 +-
ui/package.json | 34 +-
.../data-marketplace/data-marketplace.component.ts | 24 +-
.../event-schema/event-schema.component.ts | 3 +-
ui/src/app/connect/connect.module.ts | 4 +-
.../edit-correction-value.component.css} | 0
.../edit-correction-value.component.html | 39 ++
.../edit-correction-value.component.ts | 55 ++
.../edit-event-property-primitive.component.html | 1 +
.../edit-event-property-primitive.component.ts | 9 +-
.../edit-event-property.component.html | 11 +-
.../edit-event-property.component.ts | 29 +-
.../services/transformation-rule.service.ts | 41 ++
ui/src/app/core-model/coco/Annotation.ts | 2 +
ui/src/app/core-model/coco/Category.ts | 10 +-
.../app/core-model/gen/streampipes-model-client.ts | 2 +-
ui/src/app/core-model/gen/streampipes-model.ts | 322 ++++++++++-
.../datalake/datalake-rest.service.ts | 6 +-
.../semantic-type/semantic-type-utils.service.ts | 5 +
ui/src/app/core-ui/core-ui.module.ts | 118 ++--
.../image-annotations.component.html | 16 +-
.../image-annotations.component.ts | 21 +-
.../image-labels/image-labels.component.html | 39 --
.../image-labels/image-labels.component.ts | 73 ---
.../image-categorize.component.html | 5 +-
.../image-categorize/image-categorize.component.ts | 4 +-
.../image-labeling/image-labeling.component.html | 5 +-
.../image-labeling/image-labeling.component.ts | 45 +-
.../image/services/BrushLabeling.service.ts | 5 +-
.../core-ui/image/services/CocoFormat.service.ts | 22 +-
.../image/services/PolygonLabeling.service.ts | 7 +-
.../image/services/ReactLabeling.service.ts | 9 +-
.../configure-labels.component.css} | 0
.../configure-labels.component.html | 66 +++
.../configure-labels/configure-labels.component.ts | 120 +++++
.../label-list-item/label-list-item.component.css} | 0
.../label-list-item.component.html} | 27 +-
.../label-list-item/label-list-item.component.ts | 59 ++
.../select-label/select-label.component.css} | 0
.../select-label/select-label.component.html | 43 ++
.../select-label/select-label.component.ts | 81 +++
.../app/core-ui/labels/services/label.service.ts | 94 ++++
.../label-selection/label-selection.component.html | 32 --
.../label-selection/label-selection.component.ts | 61 ---
.../change-chartmode/change-chartmode.dialog.css | 29 -
.../change-chartmode/change-chartmode.dialog.ts | 35 --
.../dialogs/labeling/labeling.dialog.css | 36 --
.../dialogs/labeling/labeling.dialog.html | 36 --
.../dialogs/labeling/labeling.dialog.ts | 52 --
.../labeling-tool/labeling-tool.module.ts | 90 ----
.../filter/display-recommended.pipe.ts | 2 +-
.../static-mapping-unary.component.ts | 2 -
.../static-mapping/static-mapping.ts | 1 +
...data-explorer-dashboard-overview.component.html | 7 +-
.../data-explorer-dashboard-overview.component.ts | 7 +
.../data-explorer-dashboard-widget.component.html | 2 +-
.../data-explorer-dashboard-widget.component.ts | 2 +
.../widgets/base/base-data-explorer-widget.ts | 14 +-
.../point-selection-info.component.css | 0
.../point-selection-info.component.html | 0
.../point-selection-info.component.ts | 2 +-
.../line-chart/line-chart-widget.component.css | 5 +-
.../line-chart/line-chart-widget.component.html | 35 +-
.../line-chart/line-chart-widget.component.ts | 596 ++++++++++-----------
.../widgets/line-chart}/services/color.service.ts | 0
ui/src/app/data-explorer/data-explorer.module.ts | 96 ++--
.../save-pipeline/save-pipeline.component.html | 13 +-
.../save-pipeline/save-pipeline.component.scss | 4 +
ui/src/app/editor/editor.module.ts | 4 +-
ui/src/app/services/property-selector.service.ts | 3 +-
223 files changed, 5040 insertions(+), 2206 deletions(-)
rename streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/elements/{SendToKafkaReplayAdapterSink.java => SendToBrokerReplayAdapterSink.java} (81%)
create mode 100644 streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/preprocessing/transform/value/CorrectionValueTransformationRule.java
create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/CorrectionValueTransformationRuleDescription.java
create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/labeling/Category.java
create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/labeling/Label.java
rename streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/{init => }/NodeControllerContainer.java (93%)
copy streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/{api/NodeInfoStatusResource.java => rest/AbstractNodeContainerResource.java} (54%)
create mode 100644 streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/DebugRelayResource.java
copy streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/{api/NodeResource.java => rest/HealthCheckResource.java} (75%)
rename streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/{api/NodeInfoStatusResource.java => rest/InfoStatusResource.java} (77%)
rename streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/{init/NodeControllerContainerResourceConfig.java => rest/NodeControllerResourceConfig.java} (61%)
rename streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/{api/NodeControllerResource.java => rest/PELifeCycleResource.java} (73%)
copy ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.scss => streampipes-rest/src/main/java/org/apache/streampipes/rest/api/ICategory.java (67%)
rename streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeResource.java => streampipes-rest/src/main/java/org/apache/streampipes/rest/api/ILabel.java (64%)
create mode 100644 streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/CategoryResource.java
create mode 100644 streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/LabelResource.java
copy ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.scss => streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/ICategoryStorage.java (66%)
copy ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.scss => streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/ILabelStorage.java (64%)
create mode 100644 streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/CategoryStorageImpl.java
create mode 100644 streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/LabelStorageImpl.java
copy streampipes-wrapper-flink/src/main/java/org/apache/streampipes/wrapper/flink/consumer/{JmsConsumer.java => JmsFlinkConsumer.java} (89%)
rename streampipes-wrapper-flink/src/main/java/org/apache/streampipes/wrapper/flink/consumer/{JmsConsumer.java => MqttFlinkConsumer.java} (74%)
copy streampipes-wrapper-flink/src/main/java/org/apache/streampipes/wrapper/flink/sink/{FlinkJmsProducer.java => JmsFlinkProducer.java} (94%)
rename streampipes-wrapper-flink/src/main/java/org/apache/streampipes/wrapper/flink/sink/{FlinkJmsProducer.java => MqttFlinkProducer.java} (78%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/constants/SiddhiConstants.java
copy streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQUtils.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/constants/SiddhiStreamSelector.java (69%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/engine/SiddhiEngine.java
copy streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/engine/{SiddhiDebugCallback.java => SiddhiStatementGenerator.java} (70%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/engine/StreamPipesSiddhiProcessor.java
copy streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/engine/{ => callback}/SiddhiDebugCallback.java (93%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/engine/callback/SiddhiOutputStreamCallback.java
rename streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/engine/{SiddhiDebugCallback.java => callback/SiddhiOutputStreamDebugCallback.java} (55%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/engine/generator/EventTypeGenerator.java
copy streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/engine/generator/InputStreamNameGenerator.java (50%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/engine/generator/SiddhiAppGenerator.java
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/engine/generator/SiddhiInvocationConfigGenerator.java
copy ui/src/app/core-model/coco/Category.ts => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/model/EventPropertyDef.java (59%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/model/SiddhiProcessorParams.java
copy streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQUtils.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/AbstractQueryGenerator.java (61%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/FromClause.java
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/SelectClause.java
copy streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/SiddhiStatement.java (58%)
copy ui/src/app/core-model/coco/Category.ts => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/EqualsExpression.java (74%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/Expression.java
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/Expressions.java
copy ui/src/app/core-model/coco/Category.ts => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/GreaterEqualsExpression.java (73%)
copy ui/src/app/core-model/coco/Category.ts => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/GreaterThanExpression.java (73%)
copy ui/src/app/core-model/coco/Category.ts => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/LesserEqualsExpression.java (73%)
copy ui/src/app/core-model/coco/Category.ts => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/LesserThanExpression.java (74%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/PropertyExpression.java
copy ui/src/app/core-ui/linechart/labeling-tool/components/label-selection/label-selection.component.css => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/PropertyExpressionBase.java (85%)
copy streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/PropertyRenameExpression.java (51%)
copy streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQUtils.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/RelationalOperator.java (68%)
copy streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/RelationalOperatorExpression.java (50%)
copy streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/SequenceExpression.java (57%)
copy streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQUtils.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/SiddhiTimeUnit.java (70%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/StreamExpression.java
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/StreamFilterExpression.java
copy streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/WithinExpression.java (57%)
copy streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/list/CollectListExpression.java (51%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/list/ContainsListExpression.java
copy streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQUtils.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/math/MathAddExpression.java (70%)
copy streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQUtils.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/math/MathDivideExpression.java (70%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/math/MathExpression.java
copy streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQUtils.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/math/MathMultiplyExpression.java (69%)
copy ui/src/app/core-model/coco/Category.ts => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/math/MathOperator.java (72%)
copy streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQUtils.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/math/MathSubtractExpression.java (69%)
copy streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/pattern/EveryExpression.java (58%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/pattern/PatternCountExpression.java
rename ui/src/app/core-ui/linechart/labeling-tool/components/label-selection/label-selection.component.css => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/pattern/PatternCountOperator.java (85%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/pattern/PatternExpression.java
copy streampipes-messaging-jms/src/main/java/org/apache/streampipes/messaging/jms/ActiveMQUtils.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/window/BatchWindowExpression.java (64%)
copy streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/window/TimeWindowExpression.java (55%)
copy streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java => streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/query/expression/window/WindowExpression.java (53%)
create mode 100644 streampipes-wrapper-siddhi/src/main/java/org/apache/streampipes/wrapper/siddhi/utils/SiddhiUtils.java
copy streampipes-config/src/main/java/org/apache/streampipes/config/backend/SpProtocol.java => streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/ProcessorParams.java (56%)
copy ui/src/app/core-model/coco/Category.ts => streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/SinkParams.java (58%)
create mode 100644 streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/StreamPipesDataProcessor.java
copy ui/src/app/core-model/coco/Annotation.ts => streampipes-wrapper-standalone/src/main/java/org/apache/streampipes/wrapper/standalone/StreamPipesDataSink.java (51%)
copy ui/src/app/{editor/components/pipeline/pipeline.component.css => connect/dialog/edit-event-property/components/edit-correction-value/edit-correction-value.component.css} (100%)
create mode 100644 ui/src/app/connect/dialog/edit-event-property/components/edit-correction-value/edit-correction-value.component.html
create mode 100644 ui/src/app/connect/dialog/edit-event-property/components/edit-correction-value/edit-correction-value.component.ts
delete mode 100644 ui/src/app/core-ui/image/components/image-labels/image-labels.component.html
delete mode 100644 ui/src/app/core-ui/image/components/image-labels/image-labels.component.ts
copy ui/src/app/{editor/components/pipeline/pipeline.component.css => core-ui/labels/components/configure-labels/configure-labels.component.css} (100%)
create mode 100644 ui/src/app/core-ui/labels/components/configure-labels/configure-labels.component.html
create mode 100644 ui/src/app/core-ui/labels/components/configure-labels/configure-labels.component.ts
copy ui/src/app/{editor/components/pipeline/pipeline.component.css => core-ui/labels/components/label-list-item/label-list-item.component.css} (100%)
rename ui/src/app/core-ui/{linechart/labeling-tool/dialogs/change-chartmode/change-chartmode.dialog.html => labels/components/label-list-item/label-list-item.component.html} (53%)
create mode 100644 ui/src/app/core-ui/labels/components/label-list-item/label-list-item.component.ts
rename ui/src/app/core-ui/{image/components/image-labels/image-labels.component.css => labels/components/select-label/select-label.component.css} (100%)
create mode 100644 ui/src/app/core-ui/labels/components/select-label/select-label.component.html
create mode 100644 ui/src/app/core-ui/labels/components/select-label/select-label.component.ts
create mode 100644 ui/src/app/core-ui/labels/services/label.service.ts
delete mode 100644 ui/src/app/core-ui/linechart/labeling-tool/components/label-selection/label-selection.component.html
delete mode 100644 ui/src/app/core-ui/linechart/labeling-tool/components/label-selection/label-selection.component.ts
delete mode 100644 ui/src/app/core-ui/linechart/labeling-tool/dialogs/change-chartmode/change-chartmode.dialog.css
delete mode 100644 ui/src/app/core-ui/linechart/labeling-tool/dialogs/change-chartmode/change-chartmode.dialog.ts
delete mode 100644 ui/src/app/core-ui/linechart/labeling-tool/dialogs/labeling/labeling.dialog.css
delete mode 100644 ui/src/app/core-ui/linechart/labeling-tool/dialogs/labeling/labeling.dialog.html
delete mode 100644 ui/src/app/core-ui/linechart/labeling-tool/dialogs/labeling/labeling.dialog.ts
delete mode 100644 ui/src/app/core-ui/linechart/labeling-tool/labeling-tool.module.ts
rename ui/src/app/{core-ui/linechart/labeling-tool => data-explorer/components/widgets/line-chart}/components/point-selection-info/point-selection-info.component.css (100%)
rename ui/src/app/{core-ui/linechart/labeling-tool => data-explorer/components/widgets/line-chart}/components/point-selection-info/point-selection-info.component.html (100%)
rename ui/src/app/{core-ui/linechart/labeling-tool => data-explorer/components/widgets/line-chart}/components/point-selection-info/point-selection-info.component.ts (96%)
rename ui/src/app/{core-ui/linechart/labeling-tool => data-explorer/components/widgets/line-chart}/services/color.service.ts (100%)
[incubator-streampipes] 01/03: Merge branch 'dev' into
edge-extensions
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wiener pushed a commit to branch edge-extensions
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit fb0521e35575b245c24cadf48dde58ce528f29d2
Merge: 505c4c5 1040204
Author: Patrick Wiener <wi...@fzi.de>
AuthorDate: Tue Nov 17 20:45:55 2020 +0100
Merge branch 'dev' into edge-extensions
.github/workflows/build.yml | 38 +-
.gitlab-ci.yml | 2 +-
.../all_pipeline_elements_jvm.xml | 1 +
.idea/runConfigurations/backend.xml | 1 +
.idea/runConfigurations/sinks_internal_jvm.xml | 2 +-
Jenkinsfile | 6 +-
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../archetype-resources/src/main/java/Init.java | 1 +
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 5 +
.../archetype-resources/src/main/java/Init.java | 1 +
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../archetype-resources/src/main/java/Init.java | 1 +
.../main/resources/archetype-resources/Dockerfile | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 5 +
.../archetype-resources/src/main/java/Init.java | 1 +
.../main/resources/archetype-resources/Dockerfile | 2 +-
docker-compose.yml | 4 +-
pom.xml | 1 +
streampipes-backend/Dockerfile | 2 +-
.../backend/StreamPipesResourceConfig.java | 2 +
streampipes-backend/src/main/resources/shiro.ini | 7 +
.../commons/exceptions/SpException.java | 2 +-
.../streampipes/config/backend/BackendConfig.java | 5 +-
.../config/backend/BackendConfigKeys.java | 2 +
.../config/backend/MessagingSettings.java | 33 +-
.../streampipes/config/backend/SpProtocol.java | 10 +-
streampipes-connect-container-master/Dockerfile | 2 +-
.../streampipes/connect/adapter/Adapter.java | 60 ++-
.../connect/adapter/GroundingService.java | 36 +-
.../elements/SendToBrokerAdapterSink.java | 4 +-
...ink.java => SendToBrokerReplayAdapterSink.java} | 22 +-
.../elements/SendToJmsAdapterSink.java | 10 +-
.../elements/SendToKafkaAdapterSink.java | 6 +-
.../elements/SendToMqttAdapterSink.java | 14 +-
.../TransformValueAdapterPipelineElement.java | 7 +-
.../value/CorrectionValueTransformationRule.java | 93 ++++
.../transform/value/ValueEventTransformer.java | 38 +-
.../connect/adapter/GroundingServiceTest.java | 128 +++--
.../messaging/jms/ActiveMQPublisher.java | 15 +-
.../streampipes/messaging/jms/ActiveMQUtils.java | 12 +-
.../streampipes/messaging/mqtt/MqttConsumer.java | 13 +-
.../apache/streampipes/model/client/Category.java | 3 +-
.../rules/TransformationRuleDescription.java | 10 +-
...rrectionValueTransformationRuleDescription.java | 80 +++
.../value/ValueTransformationRuleDescription.java | 1 +
.../streampipes/model/labeling/Category.java | 67 +++
.../apache/streampipes/model/labeling/Label.java | 73 +++
.../org/apache/streampipes/model/util/Cloner.java | 9 +-
streampipes-pipeline-management/pom.xml | 5 +
.../matching/PipelineVerificationHandler.java | 3 +
.../runtime/PipelineElementRuntimeInfoFetcher.java | 28 +-
.../manager/setup/CouchDbInstallationStep.java | 24 +
.../org/apache/streampipes/rest/api/ICategory.java | 28 +-
.../org/apache/streampipes/rest/api/ILabel.java | 31 +-
.../streampipes/rest/impl/CategoryResource.java | 116 ++++
.../streampipes/rest/impl/LabelResource.java | 144 +++++
.../rest/impl/datalake/DataLakeManagementV3.java | 4 +-
.../rest/impl/datalake/DataLakeResourceV3.java | 9 +-
.../sdk/extractor/AbstractParameterExtractor.java | 39 ++
.../apache/streampipes/sdk/helpers/Protocols.java | 21 +-
.../sdk/helpers/SupportedProtocols.java | 10 +
.../apache/streampipes/sdk/utils/Datatypes.java | 8 +
.../serializers/json/GsonSerializer.java | 8 +-
.../jsonld/CustomAnnotationProvider.java | 7 +-
.../streampipes/storage/api/ICategoryStorage.java | 27 +-
.../streampipes/storage/api/ILabelStorage.java | 31 +-
.../streampipes/storage/api/INoSqlStorage.java | 5 +
.../storage/couchdb/CouchDbStorageManager.java | 6 +
.../storage/couchdb/impl/CategoryStorageImpl.java | 57 ++
.../storage/couchdb/impl/LabelStorageImpl.java | 76 +++
.../streampipes/storage/couchdb/utils/Utils.java | 12 +
.../apache/streampipes/vocabulary/StreamPipes.java | 4 +-
streampipes-wrapper-distributed/pom.xml | 5 +
.../distributed/runtime/DistributedRuntime.java | 21 +-
.../wrapper/flink/FlinkDataProcessorRuntime.java | 10 +-
.../streampipes/wrapper/flink/FlinkRuntime.java | 22 +-
.../{JmsConsumer.java => JmsFlinkConsumer.java} | 6 +-
.../{JmsConsumer.java => MqttFlinkConsumer.java} | 26 +-
...FlinkJmsProducer.java => JmsFlinkProducer.java} | 4 +-
...linkJmsProducer.java => MqttFlinkProducer.java} | 14 +-
streampipes-wrapper-siddhi/pom.xml | 6 +
.../wrapper/siddhi/constants/SiddhiConstants.java | 55 ++
.../siddhi/constants/SiddhiStreamSelector.java | 17 +-
.../wrapper/siddhi/engine/SiddhiEngine.java | 127 +++++
.../wrapper/siddhi/engine/SiddhiEventEngine.java | 274 +---------
...Callback.java => SiddhiStatementGenerator.java} | 9 +-
.../siddhi/engine/StreamPipesSiddhiProcessor.java | 57 ++
.../engine/{ => callback}/SiddhiDebugCallback.java | 2 +-
.../callback/SiddhiOutputStreamCallback.java | 54 ++
.../SiddhiOutputStreamDebugCallback.java} | 24 +-
.../engine/generator/EventTypeGenerator.java | 77 +++
.../engine/generator/InputStreamNameGenerator.java | 33 +-
.../engine/generator/SiddhiAppGenerator.java | 86 +++
.../generator/SiddhiInvocationConfigGenerator.java | 65 +++
.../wrapper/siddhi/model/EventPropertyDef.java | 26 +-
.../siddhi/model/SiddhiProcessorParams.java | 100 ++++
.../siddhi/query/AbstractQueryGenerator.java | 20 +-
.../wrapper/siddhi/query/FromClause.java | 48 ++
.../wrapper/siddhi/query/SelectClause.java | 78 +++
.../wrapper/siddhi/query/SiddhiStatement.java | 28 +-
.../siddhi/query/expression/EqualsExpression.java | 16 +-
.../siddhi/query/expression/Expression.java | 63 +++
.../siddhi/query/expression/Expressions.java | 157 ++++++
.../query/expression/GreaterEqualsExpression.java | 16 +-
.../query/expression/GreaterThanExpression.java | 16 +-
.../query/expression/LesserEqualsExpression.java | 16 +-
.../query/expression/LesserThanExpression.java | 16 +-
.../query/expression/PropertyExpression.java | 48 ++
.../query/expression/PropertyExpressionBase.java | 6 +-
.../query/expression/PropertyRenameExpression.java | 30 +-
.../query/expression/RelationalOperator.java | 22 +-
.../expression/RelationalOperatorExpression.java | 34 +-
.../query/expression/SequenceExpression.java | 24 +-
.../siddhi/query/expression/SiddhiTimeUnit.java | 18 +-
.../siddhi/query/expression/StreamExpression.java | 58 ++
.../query/expression/StreamFilterExpression.java | 57 ++
.../siddhi/query/expression/WithinExpression.java | 29 +-
.../expression/list/CollectListExpression.java | 31 +-
.../expression/list/ContainsListExpression.java | 52 ++
.../query/expression/math/MathAddExpression.java | 10 +-
.../expression/math/MathDivideExpression.java | 10 +-
.../query/expression/math/MathExpression.java | 44 ++
.../expression/math/MathMultiplyExpression.java | 10 +-
.../siddhi/query/expression/math/MathOperator.java | 23 +-
.../expression/math/MathSubtractExpression.java | 10 +-
.../query/expression/pattern/EveryExpression.java | 26 +-
.../expression/pattern/PatternCountExpression.java | 54 ++
.../expression/pattern/PatternCountOperator.java | 9 +-
.../expression/pattern/PatternExpression.java | 60 +++
.../expression/window/BatchWindowExpression.java | 15 +-
.../expression/window/TimeWindowExpression.java | 29 +-
.../query/expression/window/WindowExpression.java | 28 +-
.../wrapper/siddhi/utils/SiddhiUtils.java | 85 +++
.../wrapper/standalone/ProcessorParams.java | 27 +-
.../streampipes/wrapper/standalone/SinkParams.java | 19 +-
.../standalone/StreamPipesDataProcessor.java | 35 ++
.../wrapper/standalone/StreamPipesDataSink.java | 29 +-
ui/package.json | 34 +-
.../data-marketplace/data-marketplace.component.ts | 24 +-
.../event-schema/event-schema.component.ts | 3 +-
ui/src/app/connect/connect.module.ts | 4 +-
.../edit-correction-value.component.css} | 4 +-
.../edit-correction-value.component.html | 39 ++
.../edit-correction-value.component.ts | 55 ++
.../edit-event-property-primitive.component.html | 1 +
.../edit-event-property-primitive.component.ts | 9 +-
.../edit-event-property.component.html | 11 +-
.../edit-event-property.component.ts | 29 +-
.../services/transformation-rule.service.ts | 41 ++
ui/src/app/core-model/coco/Annotation.ts | 2 +
ui/src/app/core-model/coco/Category.ts | 10 +-
ui/src/app/core-model/gen/streampipes-model.ts | 104 ++--
.../datalake/datalake-rest.service.ts | 6 +-
.../semantic-type/semantic-type-utils.service.ts | 5 +
ui/src/app/core-ui/core-ui.module.ts | 118 ++--
.../image-annotations.component.html | 16 +-
.../image-annotations.component.ts | 21 +-
.../image-labels/image-labels.component.html | 39 --
.../image-labels/image-labels.component.ts | 73 ---
.../image-categorize.component.html | 5 +-
.../image-categorize/image-categorize.component.ts | 4 +-
.../image-labeling/image-labeling.component.html | 5 +-
.../image-labeling/image-labeling.component.ts | 45 +-
.../image/services/BrushLabeling.service.ts | 5 +-
.../core-ui/image/services/CocoFormat.service.ts | 22 +-
.../image/services/PolygonLabeling.service.ts | 7 +-
.../image/services/ReactLabeling.service.ts | 9 +-
.../configure-labels.component.css} | 4 +-
.../configure-labels.component.html | 66 +++
.../configure-labels/configure-labels.component.ts | 120 +++++
.../label-list-item/label-list-item.component.css} | 4 +-
.../label-list-item.component.html} | 27 +-
.../label-list-item/label-list-item.component.ts | 59 ++
.../select-label/select-label.component.css} | 0
.../select-label/select-label.component.html | 43 ++
.../select-label/select-label.component.ts | 81 +++
.../app/core-ui/labels/services/label.service.ts | 94 ++++
.../label-selection/label-selection.component.html | 32 --
.../label-selection/label-selection.component.ts | 61 ---
.../change-chartmode/change-chartmode.dialog.css | 29 -
.../change-chartmode/change-chartmode.dialog.ts | 35 --
.../dialogs/labeling/labeling.dialog.css | 36 --
.../dialogs/labeling/labeling.dialog.html | 36 --
.../dialogs/labeling/labeling.dialog.ts | 52 --
.../labeling-tool/labeling-tool.module.ts | 90 ----
.../filter/display-recommended.pipe.ts | 2 +-
.../static-mapping-unary.component.ts | 2 -
.../static-mapping/static-mapping.ts | 1 +
...data-explorer-dashboard-overview.component.html | 7 +-
.../data-explorer-dashboard-overview.component.ts | 7 +
.../data-explorer-dashboard-widget.component.html | 2 +-
.../data-explorer-dashboard-widget.component.ts | 2 +
.../widgets/base/base-data-explorer-widget.ts | 14 +-
.../point-selection-info.component.css | 0
.../point-selection-info.component.html | 0
.../point-selection-info.component.ts | 2 +-
.../line-chart/line-chart-widget.component.css | 5 +-
.../line-chart/line-chart-widget.component.html | 35 +-
.../line-chart/line-chart-widget.component.ts | 596 ++++++++++-----------
.../widgets/line-chart}/services/color.service.ts | 0
ui/src/app/data-explorer/data-explorer.module.ts | 96 ++--
ui/src/app/services/property-selector.service.ts | 3 +-
203 files changed, 4492 insertions(+), 1975 deletions(-)
diff --cc .github/workflows/build.yml
index 213fdd6,2dc00ac..e91b6be
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@@ -85,30 -85,11 +85,30 @@@ jobs
IMG_NAME_ARM32V7: ${{ env.DOCKERHUB_APACHE_REPO }}/streampipes-connect-container:arm32v7-${{ env.MVN_VERSION }}
IMG_NAME_ARM64V8: ${{ env.DOCKERHUB_APACHE_REPO }}/streampipes-connect-container:arm64v8-${{ env.MVN_VERSION }}
run: |
- docker build --pull --build-arg BASE_IMG=$BASE_IMG_JRE_DEFAULT -t $IMG_NAME_DEFAULT -t $IMG_NAME_AMD64 -f Dockerfile .
- docker build --pull --build-arg BASE_IMG=$BASE_IMG_JRE_ARM32V7 -t $IMG_NAME_ARM32V7 -f arm.Dockerfile .
- docker build --pull --build-arg BASE_IMG=$BASE_IMG_JRE_ARM64V8 -t $IMG_NAME_ARM64V8 -f aarch64.Dockerfile .
+ docker build --pull --build-arg BASE_IMAGE=$BASE_IMG_JRE_DEFAULT -t $IMG_NAME_DEFAULT -t $IMG_NAME_AMD64 -f Dockerfile .
+ docker build --pull --build-arg BASE_IMAGE=$BASE_IMG_JRE_ARM32V7 -t $IMG_NAME_ARM32V7 -f arm.Dockerfile .
+ docker build --pull --build-arg BASE_IMAGE=$BASE_IMG_JRE_ARM64V8 -t $IMG_NAME_ARM64V8 -f aarch64.Dockerfile .
docker push $IMG_NAME_DEFAULT
docker push $IMG_NAME_AMD64
+ docker push $IMG_NAME_ARM32V7
+ docker push $IMG_NAME_ARM64V8
+ docker manifest create $IMG_NAME_DEFAULT $IMG_NAME_AMD64 $IMG_NAME_ARM32V7 $IMG_NAME_ARM64V8
+ docker manifest annotate $IMG_NAME_DEFAULT $IMG_NAME_ARM32V7 --os linux --arch arm
+ docker manifest annotate $IMG_NAME_DEFAULT $IMG_NAME_ARM64V8 --os linux --arch arm64
+ docker manifest push $IMG_NAME_DEFAULT
+ - name: Build and Push Docker Node Controller Image
+ working-directory: ./streampipes-node-controller-container
+ env:
+ IMG_NAME_DEFAULT: ${{ env.DOCKERHUB_APACHE_REPO }}/node-controller:${{ env.MVN_VERSION }}
+ IMG_NAME_AMD64: ${{ env.DOCKERHUB_APACHE_REPO }}/node-controller:amd64-${{ env.MVN_VERSION }}
+ IMG_NAME_ARM32V7: ${{ env.DOCKERHUB_APACHE_REPO }}/node-controller:arm32v7-${{ env.MVN_VERSION }}
+ IMG_NAME_ARM64V8: ${{ env.DOCKERHUB_APACHE_REPO }}/node-controller:arm64v8-${{ env.MVN_VERSION }}
+ run: |
+ docker build --pull --build-arg BASE_IMG=$BASE_IMG_JRE_DEFAULT -t $IMG_NAME_DEFAULT -t $IMG_NAME_AMD64 -f Dockerfile .
+ docker build --pull --build-arg BASE_IMG=$BASE_IMG_JRE_ARM32V7 -t $IMG_NAME_ARM32V7 -f arm.Dockerfile .
+ docker build --pull --build-arg BASE_IMG=$BASE_IMG_JRE_ARM64V8 -t $IMG_NAME_ARM64V8 -f aarch64.Dockerfile .
+ docker push $IMG_NAME_DEFAULT
+ docker push $IMG_NAME_AMD64
docker push $IMG_NAME_ARM32V7
docker push $IMG_NAME_ARM64V8
docker manifest create $IMG_NAME_DEFAULT $IMG_NAME_AMD64 $IMG_NAME_ARM32V7 $IMG_NAME_ARM64V8
diff --cc streampipes-model/src/main/java/org/apache/streampipes/model/util/Cloner.java
index c1e7a77,c1f5e5b..178ebf7
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/util/Cloner.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/util/Cloner.java
@@@ -18,9 -18,9 +18,10 @@@
package org.apache.streampipes.model.util;
+ import org.apache.streampipes.model.grounding.*;
import org.apache.streampipes.model.output.*;
import org.apache.streampipes.model.staticproperty.*;
+import org.apache.streampipes.model.grounding.MqttTransportProtocol;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.streampipes.model.ApplicationLink;
diff --cc streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/runtime/PipelineElementRuntimeInfoFetcher.java
index 48ba076,a40ac03..93bb623
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/runtime/PipelineElementRuntimeInfoFetcher.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/runtime/PipelineElementRuntimeInfoFetcher.java
@@@ -17,7 -17,8 +17,6 @@@
*/
package org.apache.streampipes.manager.runtime;
--import com.google.inject.internal.cglib.core.$LocalVariablesSorter;
-import org.apache.streampipes.config.backend.BackendConfig;
import org.apache.streampipes.messaging.kafka.SpKafkaConsumer;
import org.apache.streampipes.messaging.mqtt.MqttConsumer;
import org.apache.streampipes.model.grounding.MqttTransportProtocol;
[incubator-streampipes] 02/03: use updated model
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wiener pushed a commit to branch edge-extensions
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit f1116ff513a2dda11f68098469f645ed49f64c86
Author: Patrick Wiener <wi...@fzi.de>
AuthorDate: Tue Nov 17 20:47:15 2020 +0100
use updated model
---
.../app/core-model/gen/streampipes-model-client.ts | 2 +-
ui/src/app/core-model/gen/streampipes-model.ts | 248 ++++++++++++++++++++-
2 files changed, 248 insertions(+), 2 deletions(-)
diff --git a/ui/src/app/core-model/gen/streampipes-model-client.ts b/ui/src/app/core-model/gen/streampipes-model-client.ts
index d10df05..8962d8a 100644
--- a/ui/src/app/core-model/gen/streampipes-model-client.ts
+++ b/ui/src/app/core-model/gen/streampipes-model-client.ts
@@ -19,7 +19,7 @@
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
-// Generated using typescript-generator version 2.24.612 on 2020-09-23 15:34:40.
+// Generated using typescript-generator version 2.24.612 on 2020-11-17 20:41:04.
export class FileMetadata {
createdAt: number;
diff --git a/ui/src/app/core-model/gen/streampipes-model.ts b/ui/src/app/core-model/gen/streampipes-model.ts
index e9e95be..1481060 100644
--- a/ui/src/app/core-model/gen/streampipes-model.ts
+++ b/ui/src/app/core-model/gen/streampipes-model.ts
@@ -19,7 +19,7 @@
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
-// Generated using typescript-generator version 2.24.612 on 2020-11-10 15:19:54.
+// Generated using typescript-generator version 2.24.612 on 2020-11-17 20:40:59.
export class AbstractStreamPipesEntity {
"@class": "org.apache.streampipes.model.base.NamedStreamPipesEntity" | "org.apache.streampipes.model.connect.adapter.AdapterDescription" | "org.apache.streampipes.model.connect.adapter.AdapterSetDescription" | "org.apache.streampipes.model.connect.adapter.GenericAdapterSetDescription" | "org.apache.streampipes.model.connect.adapter.SpecificAdapterSetDescription" | "org.apache.streampipes.model.connect.adapter.AdapterStreamDescription" | "org.apache.streampipes.model.connect.adapter.G [...]
@@ -34,6 +34,25 @@ export class AbstractStreamPipesEntity {
}
}
+export class AccessibleSensorActuatorResource {
+ connectionInfo: string;
+ connectionType: string;
+ name: string;
+ type: string;
+
+ static fromData(data: AccessibleSensorActuatorResource, target?: AccessibleSensorActuatorResource): AccessibleSensorActuatorResource {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new AccessibleSensorActuatorResource();
+ instance.name = data.name;
+ instance.type = data.type;
+ instance.connectionInfo = data.connectionInfo;
+ instance.connectionType = data.connectionType;
+ return instance;
+ }
+}
+
export class UnnamedStreamPipesEntity extends AbstractStreamPipesEntity {
"@class": "org.apache.streampipes.model.base.UnnamedStreamPipesEntity" | "org.apache.streampipes.model.connect.guess.GuessSchema" | "org.apache.streampipes.model.connect.rules.TransformationRuleDescription" | "org.apache.streampipes.model.connect.rules.value.ValueTransformationRuleDescription" | "org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription" | "org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription" | "org.apache.streamp [...]
elementId: string;
@@ -596,6 +615,21 @@ export class ApplicationLink extends UnnamedStreamPipesEntity {
}
}
+export class CPU {
+ arch: string;
+ cores: number;
+
+ static fromData(data: CPU, target?: CPU): CPU {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new CPU();
+ instance.cores = data.cores;
+ instance.arch = data.arch;
+ return instance;
+ }
+}
+
export class Category {
_id: string;
_rev: string;
@@ -717,6 +751,21 @@ export class CreateNestedRuleDescription extends SchemaTransformationRuleDescrip
}
}
+export class Cuda {
+ cudaDriverVersion: string;
+ cudaRuntimeVersion: string;
+
+ static fromData(data: Cuda, target?: Cuda): Cuda {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new Cuda();
+ instance.cudaDriverVersion = data.cudaDriverVersion;
+ instance.cudaRuntimeVersion = data.cudaRuntimeVersion;
+ return instance;
+ }
+}
+
export class CustomOutputStrategy extends OutputStrategy {
"@class": "org.apache.streampipes.model.output.CustomOutputStrategy";
availablePropertyKeys: string[];
@@ -751,6 +800,19 @@ export class CustomTransformOutputStrategy extends OutputStrategy {
}
}
+export class DISK {
+ diskTotal: number;
+
+ static fromData(data: DISK, target?: DISK): DISK {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new DISK();
+ instance.diskTotal = data.diskTotal;
+ return instance;
+ }
+}
+
export class DashboardEntity extends UnnamedStreamPipesEntity {
"@class": "org.apache.streampipes.model.dashboard.DashboardEntity" | "org.apache.streampipes.model.dashboard.DashboardWidgetModel" | "org.apache.streampipes.model.dashboard.VisualizablePipeline" | "org.apache.streampipes.model.datalake.DataExplorerWidgetModel";
_id: string;
@@ -930,6 +992,13 @@ export class InvocableStreamPipesEntity extends NamedStreamPipesEntity {
configured: boolean;
correspondingPipeline: string;
correspondingUser: string;
+ deploymentRunningInstanceId: string;
+ deploymentTargetNodeHostname: string;
+ deploymentTargetNodeId: string;
+ deploymentTargetNodePort: number;
+ elementEndpointHostname: string;
+ elementEndpointPort: number;
+ elementEndpointServiceName: string;
inputStreams: SpDataStreamUnion[];
staticProperties: StaticPropertyUnion[];
statusInfoSettings: ElementStatusInfoSettings;
@@ -951,6 +1020,13 @@ export class InvocableStreamPipesEntity extends NamedStreamPipesEntity {
instance.correspondingPipeline = data.correspondingPipeline;
instance.correspondingUser = data.correspondingUser;
instance.streamRequirements = __getCopyArrayFn(SpDataStream.fromDataUnion)(data.streamRequirements);
+ instance.elementEndpointHostname = data.elementEndpointHostname;
+ instance.elementEndpointPort = data.elementEndpointPort;
+ instance.deploymentTargetNodeId = data.deploymentTargetNodeId;
+ instance.deploymentTargetNodeHostname = data.deploymentTargetNodeHostname;
+ instance.deploymentTargetNodePort = data.deploymentTargetNodePort;
+ instance.deploymentRunningInstanceId = data.deploymentRunningInstanceId;
+ instance.elementEndpointServiceName = data.elementEndpointServiceName;
instance.configured = data.configured;
instance.uncompleted = data.uncompleted;
return instance;
@@ -1092,6 +1168,25 @@ export class DeleteRuleDescription extends SchemaTransformationRuleDescription {
}
}
+export class Docker {
+ apiVersion: string;
+ hasDocker: boolean;
+ hasNvidiaRuntime: boolean;
+ serverVersion: string;
+
+ static fromData(data: Docker, target?: Docker): Docker {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new Docker();
+ instance.hasDocker = data.hasDocker;
+ instance.hasNvidiaRuntime = data.hasNvidiaRuntime;
+ instance.serverVersion = data.serverVersion;
+ instance.apiVersion = data.apiVersion;
+ return instance;
+ }
+}
+
export class DomainPropertyProbability extends UnnamedStreamPipesEntity {
"@class": "org.apache.streampipes.model.connect.guess.DomainPropertyProbability";
domainProperty: string;
@@ -1580,6 +1675,23 @@ export class Frequency extends EventStreamQualityDefinition {
}
}
+export class GPU {
+ cudaCores: number;
+ hasGPU: boolean;
+ type: string;
+
+ static fromData(data: GPU, target?: GPU): GPU {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new GPU();
+ instance.hasGPU = data.hasGPU;
+ instance.cudaCores = data.cudaCores;
+ instance.type = data.type;
+ return instance;
+ }
+}
+
export interface GenericAdapterDescription {
eventSchema: EventSchema;
formatDescription: FormatDescription;
@@ -1642,6 +1754,25 @@ export class GuessSchema extends UnnamedStreamPipesEntity {
}
}
+export class HardwareResource {
+ cpu: CPU;
+ disk: DISK;
+ gpu: GPU;
+ memory: MEM;
+
+ static fromData(data: HardwareResource, target?: HardwareResource): HardwareResource {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new HardwareResource();
+ instance.cpu = CPU.fromData(data.cpu);
+ instance.memory = MEM.fromData(data.memory);
+ instance.disk = DISK.fromData(data.disk);
+ instance.gpu = GPU.fromData(data.gpu);
+ return instance;
+ }
+}
+
export class TransportProtocol extends UnnamedStreamPipesEntity {
"@class": "org.apache.streampipes.model.grounding.TransportProtocol" | "org.apache.streampipes.model.grounding.JmsTransportProtocol" | "org.apache.streampipes.model.grounding.KafkaTransportProtocol" | "org.apache.streampipes.model.grounding.MqttTransportProtocol";
brokerHostname: string;
@@ -1791,6 +1922,19 @@ export class ListOutputStrategy extends OutputStrategy {
}
}
+export class MEM {
+ memTotal: number;
+
+ static fromData(data: MEM, target?: MEM): MEM {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new MEM();
+ instance.memTotal = data.memTotal;
+ return instance;
+ }
+}
+
export class MappingProperty extends StaticProperty {
"@class": "org.apache.streampipes.model.staticproperty.MappingProperty" | "org.apache.streampipes.model.staticproperty.MappingPropertyUnary" | "org.apache.streampipes.model.staticproperty.MappingPropertyNary";
mapsFromOptions: string[];
@@ -1948,6 +2092,91 @@ export class MqttTransportProtocol extends TransportProtocol {
}
}
+export class Node {
+ nodeInfo: NodeInfo;
+
+ static fromData(data: Node, target?: Node): Node {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new Node();
+ instance.nodeInfo = NodeInfo.fromData(data.nodeInfo);
+ return instance;
+ }
+}
+
+export class NodeBrokerInfo {
+ host: string;
+ port: number;
+
+ static fromData(data: NodeBrokerInfo, target?: NodeBrokerInfo): NodeBrokerInfo {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new NodeBrokerInfo();
+ instance.host = data.host;
+ instance.port = data.port;
+ return instance;
+ }
+}
+
+export class NodeInfo {
+ nodeBrokerInfo: NodeBrokerInfo;
+ nodeControllerId: string;
+ nodeControllerPort: number;
+ nodeMetadata: NodeMetadata;
+ nodeResources: NodeResources;
+ supportedPipelineElementAppIds: string[];
+
+ static fromData(data: NodeInfo, target?: NodeInfo): NodeInfo {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new NodeInfo();
+ instance.nodeControllerId = data.nodeControllerId;
+ instance.nodeControllerPort = data.nodeControllerPort;
+ instance.nodeMetadata = NodeMetadata.fromData(data.nodeMetadata);
+ instance.nodeBrokerInfo = NodeBrokerInfo.fromData(data.nodeBrokerInfo);
+ instance.nodeResources = NodeResources.fromData(data.nodeResources);
+ instance.supportedPipelineElementAppIds = __getCopyArrayFn(__identity<string>())(data.supportedPipelineElementAppIds);
+ return instance;
+ }
+}
+
+export class NodeMetadata {
+ nodeAddress: string;
+ nodeLocationTags: string[];
+ nodeModel: string;
+
+ static fromData(data: NodeMetadata, target?: NodeMetadata): NodeMetadata {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new NodeMetadata();
+ instance.nodeAddress = data.nodeAddress;
+ instance.nodeModel = data.nodeModel;
+ instance.nodeLocationTags = __getCopyArrayFn(__identity<string>())(data.nodeLocationTags);
+ return instance;
+ }
+}
+
+export class NodeResources {
+ accessibleSensorActuatorResource: AccessibleSensorActuatorResource[];
+ hardwareResource: HardwareResource;
+ softwareResource: SoftwareResource;
+
+ static fromData(data: NodeResources, target?: NodeResources): NodeResources {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new NodeResources();
+ instance.hardwareResource = HardwareResource.fromData(data.hardwareResource);
+ instance.softwareResource = SoftwareResource.fromData(data.softwareResource);
+ instance.accessibleSensorActuatorResource = __getCopyArrayFn(AccessibleSensorActuatorResource.fromData)(data.accessibleSensorActuatorResource);
+ return instance;
+ }
+}
+
export class Notification {
additionalInformation: string;
description: string;
@@ -2464,6 +2693,23 @@ export class SimpleTopicDefinition extends TopicDefinition {
}
}
+export class SoftwareResource {
+ docker: Docker;
+ kernelVersion: string;
+ os: string;
+
+ static fromData(data: SoftwareResource, target?: SoftwareResource): SoftwareResource {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new SoftwareResource();
+ instance.os = data.os;
+ instance.kernelVersion = data.kernelVersion;
+ instance.docker = Docker.fromData(data.docker);
+ return instance;
+ }
+}
+
export class SpDataStream extends NamedStreamPipesEntity {
"@class": "org.apache.streampipes.model.SpDataStream" | "org.apache.streampipes.model.SpDataSet";
category: string[];
[incubator-streampipes] 03/03: initial changes to
InvocableGraphBuilder
Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wiener pushed a commit to branch edge-extensions
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 3588b8a1849f10e8e0a1497fd18fb12bb9067a31
Author: Patrick Wiener <wi...@fzi.de>
AuthorDate: Fri Nov 20 10:01:24 2020 +0100
initial changes to InvocableGraphBuilder
---
.../master/management/WorkerRestClient.java | 2 +-
.../streampipes/container/util/ConsulUtil.java | 10 +-
streampipes-node-controller-container/pom.xml | 3 +-
.../{init => }/NodeControllerContainer.java | 5 +-
.../controller/container/config/ConfigKeys.java | 1 -
.../container/management/info/NodeInfoStorage.java | 2 +-
.../management/resource/ResourceManager.java | 52 +++----
.../AbstractNodeContainerResource.java} | 49 ++++---
.../container/rest/DebugRelayResource.java | 55 +++++++
.../HealthCheckResource.java} | 11 +-
.../InfoStatusResource.java} | 14 +-
.../NodeControllerResourceConfig.java} | 18 +--
.../PELifeCycleResource.java} | 81 +++--------
.../http/InvocableEntityUrlGenerator.java | 4 +-
.../manager/matching/InvocationGraphBuilder.java | 161 +++++++++++++++------
.../manager/matching/ProtocolSelector.java | 44 +++---
.../apache/streampipes/vocabulary/StreamPipes.java | 1 +
.../save-pipeline/save-pipeline.component.html | 13 +-
.../save-pipeline/save-pipeline.component.scss | 4 +
ui/src/app/editor/editor.module.ts | 4 +-
20 files changed, 323 insertions(+), 211 deletions(-)
diff --git a/streampipes-connect-container-master/src/main/java/org/apache/streampipes/connect/container/master/management/WorkerRestClient.java b/streampipes-connect-container-master/src/main/java/org/apache/streampipes/connect/container/master/management/WorkerRestClient.java
index 747fdbb..3d1a4cc 100644
--- a/streampipes-connect-container-master/src/main/java/org/apache/streampipes/connect/container/master/management/WorkerRestClient.java
+++ b/streampipes-connect-container-master/src/main/java/org/apache/streampipes/connect/container/master/management/WorkerRestClient.java
@@ -263,7 +263,7 @@ public class WorkerRestClient {
private static byte[] getIconAsset(String baseUrl, String appId) throws AdapterException {
String url = baseUrl + "/" + appId + "/assets/icon";
- logger.info("Trying to get icon from endpoint: " + url);
+ logger.debug("Trying to get icon from endpoint: " + url);
try {
byte[] responseString = Request.Get(url)
diff --git a/streampipes-container/src/main/java/org/apache/streampipes/container/util/ConsulUtil.java b/streampipes-container/src/main/java/org/apache/streampipes/container/util/ConsulUtil.java
index ecfc10e..d90df3d 100644
--- a/streampipes-container/src/main/java/org/apache/streampipes/container/util/ConsulUtil.java
+++ b/streampipes-container/src/main/java/org/apache/streampipes/container/util/ConsulUtil.java
@@ -170,8 +170,8 @@ public class ConsulUtil {
}
- public static int getElementEndpointPort(String route) {
- String value = ConsulUtil.getKeyValue(route)
+ public static int getIntValue(String route) {
+ String value = getKeyValue(route)
.values()
.stream()
.findFirst()
@@ -188,8 +188,8 @@ public class ConsulUtil {
return Integer.parseInt(new Gson().fromJson(value, ConfigItem.class).getValue());
}
- public static String getElementEndpointHostname(String route) {
- String value = ConsulUtil.getKeyValue(route)
+ public static String getStringValue(String route) {
+ String value = getKeyValue(route)
.values()
.stream()
.findFirst()
@@ -241,7 +241,7 @@ public class ConsulUtil {
for (ServiceHealth node : nodes) {
if (node.getService().getTags().containsAll(filterByTags)) {
String endpoint = node.getService().getAddress() + ":" + node.getService().getPort();
- LOG.info("Active" + serviceGroup + " endpoint:" + endpoint);
+ LOG.info("Active " + serviceGroup + " endpoint: " + endpoint);
endpoints.add(endpoint);
}
}
diff --git a/streampipes-node-controller-container/pom.xml b/streampipes-node-controller-container/pom.xml
index ca770a5..9f274d6 100644
--- a/streampipes-node-controller-container/pom.xml
+++ b/streampipes-node-controller-container/pom.xml
@@ -100,7 +100,7 @@
<dependency>
<groupId>com.github.oshi</groupId>
<artifactId>oshi-core</artifactId>
- <version>4.1.1</version>
+ <version>5.3.6</version>
</dependency>
<dependency>
<groupId>org.eclipse.paho</groupId>
@@ -116,7 +116,6 @@
<groupId>org.apache.streampipes</groupId>
<artifactId>streampipes-messaging-kafka</artifactId>
<version>0.68.0-SNAPSHOT</version>
- <scope>compile</scope>
</dependency>
</dependencies>
diff --git a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/init/NodeControllerContainer.java b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/NodeControllerContainer.java
similarity index 93%
rename from streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/init/NodeControllerContainer.java
rename to streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/NodeControllerContainer.java
index 96de524..8210dc4 100644
--- a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/init/NodeControllerContainer.java
+++ b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/NodeControllerContainer.java
@@ -1,4 +1,4 @@
-package org.apache.streampipes.node.controller.container.init;
+package org.apache.streampipes.node.controller.container;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -18,6 +18,7 @@ package org.apache.streampipes.node.controller.container.init;
*/
import org.apache.streampipes.container.util.ConsulUtil;
+import org.apache.streampipes.node.controller.container.rest.NodeControllerResourceConfig;
import org.apache.streampipes.node.controller.container.config.NodeControllerConfig;
import org.apache.streampipes.node.controller.container.management.info.NodeInfoStorage;
import org.apache.streampipes.node.controller.container.management.node.NodeJanitorManager;
@@ -34,7 +35,7 @@ import java.util.Collections;
@Configuration
@EnableAutoConfiguration
-@Import({ NodeControllerContainerResourceConfig.class })
+@Import({ NodeControllerResourceConfig.class })
public class NodeControllerContainer {
private static final Logger LOG =
diff --git a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/config/ConfigKeys.java b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/config/ConfigKeys.java
index 1335ee6..e3ea78e 100644
--- a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/config/ConfigKeys.java
+++ b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/config/ConfigKeys.java
@@ -31,6 +31,5 @@ public class ConfigKeys {
final static String NODE_SUPPORTED_PE_APP_ID_KEY = "SP_NODE_SUPPORTED_PE_APP_ID";
final static String DOCKER_PRUNING_FREQ_SECS_KEY = "SP_DOCKER_PRUNING_FREQ_SECS";
final static String NODE_RESOURCE_UPDATE_FREQ_SECS_KEY = "SP_NODE_RESOURCE_UPDATE_FREQ_SECS";
-
final static String NODE_EVENT_BUFFER_SIZE = "SP_NODE_EVENT_BUFFER_SIZE";
}
diff --git a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/management/info/NodeInfoStorage.java b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/management/info/NodeInfoStorage.java
index 63a985c..246dd37 100644
--- a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/management/info/NodeInfoStorage.java
+++ b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/management/info/NodeInfoStorage.java
@@ -210,7 +210,7 @@ public class NodeInfoStorage {
}
private static Long getDiskUsage(FileSystem fs) {
- OSFileStore[] fsArray = fs.getFileStores();
+ List<OSFileStore> fsArray = fs.getFileStores();
long diskTotal = 0L;
for(OSFileStore f : fsArray) {
// has SATA disk
diff --git a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/management/resource/ResourceManager.java b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/management/resource/ResourceManager.java
index aa7a835..e9b5c31 100644
--- a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/management/resource/ResourceManager.java
+++ b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/management/resource/ResourceManager.java
@@ -65,7 +65,7 @@ public class ResourceManager {
new Thread(getCurrentResources, "rm").start();
}
- private Runnable getCurrentResources = () -> {
+ private final Runnable getCurrentResources = () -> {
while(true) {
try {
@@ -94,12 +94,12 @@ public class ResourceManager {
nodeResources.put("cpuLoadInPercent", cpuLoad);
nodeResources.put("cpuTemperature", String.format("%.2f°C", cpuTemperature));
nodeResources.put("cpuTemperatureCelcius", cpuTemperature);
- nodeResources.put("freeMemory", freeMemory);
- nodeResources.put("usedMemory", usedMemory);
- nodeResources.put("totalMemory", totalMemory);
+ nodeResources.put("freeMemoryInBytes", freeMemory);
+ nodeResources.put("usedMemoryInBytes", usedMemory);
+ nodeResources.put("totalMemoryInBytes", totalMemory);
for (Map.Entry<String, Map<String, Long>> k : diskUsage.entrySet()) {
- nodeResources.put("availableDisk", k.getValue().get("available"));
+ nodeResources.put("freeDiskSpaceInBytes", k.getValue().get("usableDiskSpace"));
}
} catch (InterruptedException e) {
@@ -118,67 +118,67 @@ public class ResourceManager {
}
private Map<String, Map<String,Long>> getDiskUsage(FileSystem fs) {
- OSFileStore[] fsArray = fs.getFileStores();
+ List<OSFileStore> fsArray = fs.getFileStores();
Map<String, Map<String, Long>> m = new HashMap<>();
for(OSFileStore f : fsArray) {
Map<String, Long> i = new HashMap<>();
// has SATA disk
if (f.getVolume().contains("/dev/sda")){
- i.put("available", f.getUsableSpace());
- i.put("total", f.getTotalSpace());
+ i.put("usableDiskSpace", f.getUsableSpace());
+ i.put("totalDiskSpace", f.getTotalSpace());
m.put(f.getVolume(), i);
}
else if (f.getVolume().contains("/dev/nvme")){
- i.put("available", f.getUsableSpace());
- i.put("total", f.getTotalSpace());
+ i.put("usableDiskSpace", f.getUsableSpace());
+ i.put("totalDiskSpace", f.getTotalSpace());
m.put(f.getVolume(), i);
}
else if (f.getVolume().contains("/dev/disk")){
- i.put("available", f.getUsableSpace());
- i.put("total", f.getTotalSpace());
+ i.put("usableDiskSpace", f.getUsableSpace());
+ i.put("totalDiskSpace", f.getTotalSpace());
m.put(f.getVolume(), i);
}
// Docker in RPi
else if (f.getVolume().contains("/dev/root")){
- i.put("available", f.getUsableSpace());
- i.put("total", f.getTotalSpace());
+ i.put("usableDiskSpace", f.getUsableSpace());
+ i.put("totalDiskSpace", f.getTotalSpace());
m.put(f.getVolume(), i);
}
// Docker in Jetson Nano
else if (f.getVolume().contains("/dev/mmcblk0p1")){
- i.put("available", f.getUsableSpace());
- i.put("total", f.getTotalSpace());
+ i.put("usableDiskSpace", f.getUsableSpace());
+ i.put("totalDiskSpace", f.getTotalSpace());
m.put(f.getVolume(), i);
}
// // has SATA disk
// if (f.getVolume().contains("/dev/sda") && ( f.getMount().equals("/") || f.getMount().equals("/home"))){
-// i.put("available", f.getUsableSpace());
-// i.put("total", f.getTotalSpace());
+// i.put("usableDiskSpace", f.getUsableSpace());
+// i.put("totalDiskSpace", f.getTotalSpace());
// m.put(f.getVolume(), i);
// }
// // has overlay disk (container setup)
// else if (f.getVolume().equals("overlay") && ( f.getMount().equals("/") || f.getMount().equals("/home"))){
-// i.put("available", f.getUsableSpace());
-// i.put("total", f.getTotalSpace());
+// i.put("usableDiskSpace", f.getUsableSpace());
+// i.put("totalDiskSpace", f.getTotalSpace());
// m.put(f.getVolume(), i);
// }
// // has NVME disk
// else if(f.getVolume().contains("/dev/nvme") && ( f.getMount().equals("/") || f.getMount().equals("/home"))) {
-// i.put("available", f.getUsableSpace());
-// i.put("total", f.getTotalSpace());
+// i.put("usableDiskSpace", f.getUsableSpace());
+// i.put("totalDiskSpace", f.getTotalSpace());
// m.put(f.getVolume(), i);
// }
// // disk on macOS
// else if (f.getVolume().contains("/dev/disk") && f.getMount().equals("/")) {
-// i.put("available", f.getUsableSpace());
-// i.put("total", f.getTotalSpace());
+// i.put("usableDiskSpace", f.getUsableSpace());
+// i.put("totalDiskSpace", f.getTotalSpace());
// m.put(f.getVolume(), i);
// }
}
if (m.isEmpty()) {
Map<String, Long> i = new HashMap<>();
- i.put("available", 0L);
- i.put("total", 0L);
+ i.put("usableDiskSpace", 0L);
+ i.put("totalDiskSpace", 0L);
m.put("n/a", i);
}
return m;
diff --git a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeInfoStatusResource.java b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/AbstractNodeContainerResource.java
similarity index 54%
copy from streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeInfoStatusResource.java
copy to streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/AbstractNodeContainerResource.java
index 628ef09..86c4444 100644
--- a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeInfoStatusResource.java
+++ b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/AbstractNodeContainerResource.java
@@ -1,4 +1,3 @@
-package org.apache.streampipes.node.controller.container.api;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -16,35 +15,47 @@ package org.apache.streampipes.node.controller.container.api;
* limitations under the License.
*
*/
-import org.apache.streampipes.node.controller.container.management.info.NodeInfoStorage;
-import org.apache.streampipes.node.controller.container.management.resource.ResourceManager;
+package org.apache.streampipes.node.controller.container.rest;
+
+import org.apache.streampipes.model.message.Message;
+import sun.security.provider.certpath.OCSPResponse;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-@Path("/node")
-public class NodeInfoStatusResource {
+public abstract class AbstractNodeContainerResource {
- @GET
- @Path("/info")
- @Produces(MediaType.APPLICATION_JSON)
- public Response getInfo() {
+ protected <T> Response ok(T entity) {
return Response
.ok()
- .entity(NodeInfoStorage.getInstance().retrieveNodeInfo())
+ .entity(entity)
.build();
}
- @GET
- @Path("/status")
- @Produces(MediaType.APPLICATION_JSON)
- public Response getStatus() {
+ protected <T> Response ok() {
return Response
.ok()
- .entity(ResourceManager.getInstance().retrieveNodeResources())
+ .build();
+ }
+
+ protected <T> Response error(T entity) {
+ return Response
+ .status(500)
+ .entity(entity)
+ .build();
+ }
+
+ protected Response statusMessage(Message message) {
+ return ok(message);
+ }
+
+ protected Response fail() {
+ return Response.serverError().build();
+ }
+
+ protected <T> Response fail(T entity) {
+ return Response
+ .serverError()
+ .entity(entity)
.build();
}
}
diff --git a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/DebugRelayResource.java b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/DebugRelayResource.java
new file mode 100644
index 0000000..0ff1ca6
--- /dev/null
+++ b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/DebugRelayResource.java
@@ -0,0 +1,55 @@
+package org.apache.streampipes.node.controller.container.rest;/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.node.controller.container.management.relay.EventRelayManager;
+import org.apache.streampipes.node.controller.container.management.relay.RunningRelayInstances;
+
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.core.Response;
+
+public class DebugRelayResource extends AbstractNodeContainerResource {
+
+ // TODO: Debug-only.
+ @POST
+ @Path("/relay/start")
+ public Response debugRelayEventStream(String msg) throws SpRuntimeException {
+ // TODO implement
+
+ System.out.println(msg);
+ EventRelayManager eventRelayManager = new EventRelayManager();
+ eventRelayManager.start();
+ RunningRelayInstances.INSTANCE.addRelay(eventRelayManager.getRelayedTopic(), eventRelayManager);
+
+ return ok();
+ }
+
+ @POST
+ @Path("/relay/stop")
+ public Response debugStopRelayEventStream(String msg) throws SpRuntimeException {
+ // TODO implement
+
+ System.out.println(msg);
+ EventRelayManager eventRelayManager = RunningRelayInstances.INSTANCE.get("org.apache.streampipes.flowrate01");
+ assert eventRelayManager != null;
+ eventRelayManager.stop();
+
+ return ok();
+ }
+}
diff --git a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeResource.java b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/HealthCheckResource.java
similarity index 75%
rename from streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeResource.java
rename to streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/HealthCheckResource.java
index 06759b3..e4dddf4 100644
--- a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeResource.java
+++ b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/HealthCheckResource.java
@@ -1,4 +1,3 @@
-package org.apache.streampipes.node.controller.container.api;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -16,6 +15,9 @@ package org.apache.streampipes.node.controller.container.api;
* limitations under the License.
*
*/
+package org.apache.streampipes.node.controller.container.rest;
+
+import org.apache.streampipes.node.controller.container.config.NodeControllerConfig;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
@@ -24,14 +26,11 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
@Path("/")
-public class NodeResource {
+public class HealthCheckResource extends AbstractNodeContainerResource{
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response getHealth() {
- return Response
- .ok()
- .status(Response.Status.OK)
- .build();
+ return ok(String.format("hello from node controller: %s", NodeControllerConfig.INSTANCE.getNodeControllerId()));
}
}
diff --git a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeInfoStatusResource.java b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/InfoStatusResource.java
similarity index 77%
rename from streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeInfoStatusResource.java
rename to streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/InfoStatusResource.java
index 628ef09..40b3812 100644
--- a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeInfoStatusResource.java
+++ b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/InfoStatusResource.java
@@ -1,4 +1,4 @@
-package org.apache.streampipes.node.controller.container.api;
+package org.apache.streampipes.node.controller.container.rest;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -26,25 +26,19 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
@Path("/node")
-public class NodeInfoStatusResource {
+public class InfoStatusResource extends AbstractNodeContainerResource{
@GET
@Path("/info")
@Produces(MediaType.APPLICATION_JSON)
public Response getInfo() {
- return Response
- .ok()
- .entity(NodeInfoStorage.getInstance().retrieveNodeInfo())
- .build();
+ return ok(NodeInfoStorage.getInstance().retrieveNodeInfo());
}
@GET
@Path("/status")
@Produces(MediaType.APPLICATION_JSON)
public Response getStatus() {
- return Response
- .ok()
- .entity(ResourceManager.getInstance().retrieveNodeResources())
- .build();
+ return ok(ResourceManager.getInstance().retrieveNodeResources());
}
}
diff --git a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/init/NodeControllerContainerResourceConfig.java b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/NodeControllerResourceConfig.java
similarity index 61%
rename from streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/init/NodeControllerContainerResourceConfig.java
rename to streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/NodeControllerResourceConfig.java
index 2704c0a..68930be 100644
--- a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/init/NodeControllerContainerResourceConfig.java
+++ b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/NodeControllerResourceConfig.java
@@ -1,4 +1,4 @@
-package org.apache.streampipes.node.controller.container.init;
+package org.apache.streampipes.node.controller.container.rest;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -17,18 +17,18 @@ package org.apache.streampipes.node.controller.container.init;
*
*/
-import org.apache.streampipes.node.controller.container.api.NodeControllerResource;
-import org.apache.streampipes.node.controller.container.api.NodeInfoStatusResource;
-import org.apache.streampipes.node.controller.container.api.NodeResource;
import org.glassfish.jersey.server.ResourceConfig;
import org.springframework.stereotype.Component;
@Component
-public class NodeControllerContainerResourceConfig extends ResourceConfig {
+public class NodeControllerResourceConfig extends ResourceConfig {
- public NodeControllerContainerResourceConfig() {
- register(NodeResource.class);
- register(NodeInfoStatusResource.class);
- register(NodeControllerResource.class);
+ public NodeControllerResourceConfig() {
+ register(HealthCheckResource.class);
+ register(InfoStatusResource.class);
+ register(PELifeCycleResource.class);
+
+ // TODO remove later - only for local relay tests
+ register(DebugRelayResource.class);
}
}
diff --git a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeControllerResource.java b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/PELifeCycleResource.java
similarity index 73%
rename from streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeControllerResource.java
rename to streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/PELifeCycleResource.java
index fb77e4b..091331b 100644
--- a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/api/NodeControllerResource.java
+++ b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/rest/PELifeCycleResource.java
@@ -1,4 +1,4 @@
-package org.apache.streampipes.node.controller.container.api;/*
+package org.apache.streampipes.node.controller.container.rest;/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,8 +17,6 @@ package org.apache.streampipes.node.controller.container.api;/*
*/
import org.apache.streampipes.commons.exceptions.SpRuntimeException;
-import org.apache.streampipes.container.declarer.InvocableDeclarer;
-import org.apache.streampipes.container.init.RunningInstances;
import org.apache.streampipes.container.transform.Transformer;
import org.apache.streampipes.container.util.Util;
import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
@@ -29,7 +27,6 @@ import org.apache.streampipes.node.controller.container.management.container.Doc
import org.apache.streampipes.node.controller.container.management.pe.PipelineElementManager;
import org.apache.streampipes.node.controller.container.management.relay.EventRelayManager;
import org.apache.streampipes.node.controller.container.management.relay.RunningRelayInstances;
-import org.eclipse.paho.client.mqttv3.MqttException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,33 +36,41 @@ import javax.ws.rs.core.Response;
import java.io.IOException;
@Path("/node/container")
-public class NodeControllerResource<I extends InvocableStreamPipesEntity> {
+public class PELifeCycleResource<I extends InvocableStreamPipesEntity> extends AbstractNodeContainerResource{
private static final Logger LOG =
- LoggerFactory.getLogger(NodeControllerResource.class.getCanonicalName());
-
+ LoggerFactory.getLogger(PELifeCycleResource.class.getCanonicalName());
private static final String COLON = ":";
+ /**
+ *
+ * @return a list of currently running Docker Containers
+ */
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response getPipelineElementContainer(){
- return Response
- .ok()
- .entity(DockerOrchestratorManager.getInstance().list())
- .build();
+ return ok(DockerOrchestratorManager.getInstance().list());
}
+ /**
+ * Deploys a new Docker Container
+ *
+ * @param container to be deployed
+ * @return deployment status
+ */
@POST
@Path("/deploy")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response deployPipelineElementContainer(PipelineElementDockerContainer container) {
- return Response
- .ok()
- .entity(DockerOrchestratorManager.getInstance().deploy(container))
- .build();
+ return ok(DockerOrchestratorManager.getInstance().deploy(container));
}
+ /**
+ * Register pipeline elements in consul
+ * @param message
+ * @return
+ */
@POST
@Path("/register")
@Consumes(MediaType.APPLICATION_JSON)
@@ -81,10 +86,7 @@ public class NodeControllerResource<I extends InvocableStreamPipesEntity> {
// .build();
// client.execute(request);
- return Response
- .ok()
- .status(Response.Status.OK)
- .build();
+ return ok();
}
@POST
@@ -117,9 +119,7 @@ public class NodeControllerResource<I extends InvocableStreamPipesEntity> {
} catch (IOException e) {
e.printStackTrace();
}
- return Response
- .ok()
- .build();
+ return ok();
}
// TODO move endpoint to /elementId/instances/runningInstanceId
@@ -144,9 +144,7 @@ public class NodeControllerResource<I extends InvocableStreamPipesEntity> {
EventRelayManager relay = RunningRelayInstances.INSTANCE.removeRelay(appId);
relay.stop();
- return Response
- .ok()
- .build();
+ return ok();
}
@DELETE
@@ -154,38 +152,7 @@ public class NodeControllerResource<I extends InvocableStreamPipesEntity> {
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response removePipelineElementContainer(PipelineElementDockerContainer container) {
- return Response
- .ok(DockerOrchestratorManager.getInstance().remove(container))
- .build();
- }
-
- // TODO: Debug-only.
- @POST
- @Path("/relay/start")
- public Response debugRelayEventStream(String msg) throws SpRuntimeException {
- // TODO implement
-
- System.out.println(msg);
- EventRelayManager eventRelayManager = new EventRelayManager();
- eventRelayManager.start();
- RunningRelayInstances.INSTANCE.addRelay(eventRelayManager.getRelayedTopic(), eventRelayManager);
-
- return Response
- .ok()
- .build();
+ return ok(DockerOrchestratorManager.getInstance().remove(container));
}
- @POST
- @Path("/relay/stop")
- public Response debugStopRelayEventStream(String msg) throws SpRuntimeException {
- // TODO implement
-
- System.out.println(msg);
- EventRelayManager eventRelayManager = RunningRelayInstances.INSTANCE.get("org.apache.streampipes.flowrate01");
- eventRelayManager.stop();
-
- return Response
- .ok()
- .build();
- }
}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/InvocableEntityUrlGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/InvocableEntityUrlGenerator.java
index 2999aca..94bd5cb 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/InvocableEntityUrlGenerator.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/InvocableEntityUrlGenerator.java
@@ -93,8 +93,8 @@ public class InvocableEntityUrlGenerator extends EndpointUrlGenerator<InvocableS
+ pipelineElement.getDeploymentTargetNodeId()
+ SLASH;
- String host = ConsulUtil.getElementEndpointHostname(route + PE_HOST_KEY);
- int port = ConsulUtil.getElementEndpointPort(route + PE_PORT_KEY);
+ String host = ConsulUtil.getStringValue(route + PE_HOST_KEY);
+ int port = ConsulUtil.getIntValue(route + PE_PORT_KEY);
// Necessary because secondary pipeline element description is not stored in backend
// It uses information from primary pipeline element. Node controller will locally forward
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/InvocationGraphBuilder.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/InvocationGraphBuilder.java
index 3f11c1a..80c9acd 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/InvocationGraphBuilder.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/InvocationGraphBuilder.java
@@ -19,6 +19,7 @@
package org.apache.streampipes.manager.matching;
import org.apache.streampipes.config.backend.BackendConfig;
+import org.apache.streampipes.container.util.ConsulUtil;
import org.apache.streampipes.manager.data.PipelineGraph;
import org.apache.streampipes.manager.data.PipelineGraphHelpers;
import org.apache.streampipes.manager.matching.output.OutputSchemaFactory;
@@ -27,7 +28,7 @@ import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
import org.apache.streampipes.model.base.NamedStreamPipesEntity;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
-import org.apache.streampipes.model.grounding.EventGrounding;
+import org.apache.streampipes.model.grounding.*;
import org.apache.streampipes.model.monitoring.ElementStatusInfoSettings;
import org.apache.streampipes.model.output.OutputStrategy;
import org.apache.streampipes.model.schema.EventSchema;
@@ -41,75 +42,97 @@ import java.util.stream.Collectors;
public class InvocationGraphBuilder {
- private PipelineGraph pipelineGraph;
- private String pipelineId;
+ private final PipelineGraph pipelineGraph;
+ private final String pipelineId;
private Integer uniquePeIndex = 0;
-
- private List<InvocableStreamPipesEntity> graphs;
+ private final List<InvocableStreamPipesEntity> graphs;
public InvocationGraphBuilder(PipelineGraph pipelineGraph, String pipelineId) {
this.graphs = new ArrayList<>();
this.pipelineGraph = pipelineGraph;
this.pipelineId = pipelineId;
-
}
public List<InvocableStreamPipesEntity> buildGraphs() {
- List<SpDataStream> streams = PipelineGraphHelpers.findStreams(pipelineGraph);
-
- for (SpDataStream stream : streams) {
- Set<InvocableStreamPipesEntity> connectedElements = getConnections(stream);
- configure(stream, connectedElements);
- }
+ PipelineGraphHelpers
+ .findStreams(pipelineGraph)
+ .forEach(stream -> configure(stream, getConnections(stream)));
return graphs;
}
private void configure(NamedStreamPipesEntity source, Set<InvocableStreamPipesEntity> targets) {
- EventGrounding inputGrounding = new GroundingBuilder(source, targets)
- .getEventGrounding();
+ EventGrounding inputGrounding = new GroundingBuilder(source, targets).getEventGrounding();
+ // set output stream event grounding for source data processors
if (source instanceof InvocableStreamPipesEntity) {
if (source instanceof DataProcessorInvocation && ((DataProcessorInvocation) source).isConfigured()) {
- DataProcessorInvocation dataProcessorInvocation = (DataProcessorInvocation) source;
- Tuple2<EventSchema, ? extends OutputStrategy> outputSettings;
- OutputSchemaGenerator<?> schemaGenerator = new OutputSchemaFactory(dataProcessorInvocation)
- .getOuputSchemaGenerator();
-
- if (((DataProcessorInvocation) source).getInputStreams().size() == 1) {
- outputSettings = schemaGenerator.buildFromOneStream(dataProcessorInvocation
- .getInputStreams()
- .get(0));
- } else if (graphExists(dataProcessorInvocation.getDOM())) {
- DataProcessorInvocation existingInvocation = (DataProcessorInvocation) find(dataProcessorInvocation.getDOM());
-
- outputSettings = schemaGenerator.buildFromTwoStreams(existingInvocation
- .getInputStreams().get(0), dataProcessorInvocation.getInputStreams().get(1));
- graphs.remove(existingInvocation);
- } else {
- outputSettings = new Tuple2<>(new EventSchema(), dataProcessorInvocation
- .getOutputStrategies().get(0));
- }
+ DataProcessorInvocation sourceInvokation = (DataProcessorInvocation) source;
- SpDataStream outputStream = new SpDataStream();
- outputStream.setEventGrounding(inputGrounding);
- dataProcessorInvocation.setOutputStrategies(Collections.singletonList(outputSettings.b));
- outputStream.setEventSchema(outputSettings.a);
- ((DataProcessorInvocation) source).setOutputStream(outputStream);
+ Tuple2<EventSchema, ? extends OutputStrategy> outputSettings = getOutputSettings(sourceInvokation);
+ sourceInvokation.setOutputStrategies(Collections.singletonList(outputSettings.b));
+ sourceInvokation.setOutputStream(makeOutputStream(inputGrounding, outputSettings));
}
-
if (!graphExists(source.getDOM())) {
graphs.add((InvocableStreamPipesEntity) source);
}
}
+ // set input stream event grounding for target element data processors and sinks
targets.forEach(t -> {
- t.getInputStreams()
- .get(getIndex(source.getDOM(), t))
- .setEventGrounding(inputGrounding);
+ // check if source and target share same node
+ if (source instanceof InvocableStreamPipesEntity) {
+ if (((InvocableStreamPipesEntity) source).getDeploymentTargetNodeId() != null ||
+ t.getDeploymentTargetNodeId() != null) {
+
+ if (matchingDeploymentTarget((InvocableStreamPipesEntity) source, t)) {
+ // shared grounding
+ // TODO: set event relay to false
+ t.getInputStreams()
+ .get(getIndex(source.getDOM(), t))
+ .setEventGrounding(inputGrounding);
+
+ } else {
+ // check if target runs on cloud or edge node
+ if (t.getDeploymentTargetNodeId().equals("default")) {
+
+ // target runs on cloud node: use central cloud broker, e.g. kafka
+ // TODO: set event relay to true
+ // TODO: add cloud broker to List<EventRelays>
+ t.getInputStreams()
+ .get(getIndex(source.getDOM(), t))
+ .setEventGrounding(inputGrounding);
+
+ } else {
+ // target runs on edge node: use target edge node broker
+ // TODO: set event relay to true
+ // TODO: add target edge node broker to List<EventRelays>
+
+ String broker = getEdgeBroker(t);
+
+ t.getInputStreams()
+ .get(getIndex(source.getDOM(), t))
+ .setEventGrounding(inputGrounding);
+ }
+ }
+ } else {
+ t.getInputStreams()
+ .get(getIndex(source.getDOM(), t))
+ .setEventGrounding(inputGrounding);
+ }
+ } else {
+ t.getInputStreams()
+ .get(getIndex(source.getDOM(), t))
+ .setEventGrounding(inputGrounding);
+ }
+
+ // old
+// t.getInputStreams()
+// .get(getIndex(source.getDOM(), t))
+// .setEventGrounding(inputGrounding);
t.getInputStreams()
.get(getIndex(source.getDOM(), t))
@@ -128,7 +151,56 @@ public class InvocationGraphBuilder {
configure(t, getConnections(t));
});
+ }
+ private Tuple2<EventSchema,? extends OutputStrategy> getOutputSettings(DataProcessorInvocation dataProcessorInvocation) {
+ Tuple2<EventSchema,? extends OutputStrategy> outputSettings;
+ OutputSchemaGenerator<?> schemaGenerator = new OutputSchemaFactory(dataProcessorInvocation)
+ .getOuputSchemaGenerator();
+
+ if (dataProcessorInvocation.getInputStreams().size() == 1) {
+ outputSettings = schemaGenerator
+ .buildFromOneStream(dataProcessorInvocation
+ .getInputStreams()
+ .get(0));
+ } else if (graphExists(dataProcessorInvocation.getDOM())) {
+ DataProcessorInvocation existingInvocation = (DataProcessorInvocation) find(dataProcessorInvocation.getDOM());
+ outputSettings = schemaGenerator
+ .buildFromTwoStreams(
+ existingInvocation.getInputStreams().get(0),
+ dataProcessorInvocation.getInputStreams().get(1));
+ graphs.remove(existingInvocation);
+ } else {
+ outputSettings = new Tuple2<>(new EventSchema(), dataProcessorInvocation.getOutputStrategies().get(0));
+ }
+ return outputSettings;
+ }
+
+ private SpDataStream makeOutputStream(EventGrounding inputGrounding,
+ Tuple2<EventSchema,? extends OutputStrategy> outputSettings) {
+ SpDataStream outputStream = new SpDataStream();
+ outputStream.setEventGrounding(inputGrounding);
+ outputStream.setEventSchema(outputSettings.a);
+ return outputStream;
+ }
+
+ private String getEdgeBroker(InvocableStreamPipesEntity target) {
+ return ConsulUtil.getStringValue(
+ "sp/v1/node/org.apache.streampipes.node.controller/"
+ + target.getDeploymentTargetNodeId()
+ + "/config/SP_NODE_BROKER_HOST");
+ }
+
+
+ private boolean matchingDeploymentTarget(InvocableStreamPipesEntity source, InvocableStreamPipesEntity target) {
+ if (source instanceof DataProcessorInvocation && target instanceof DataProcessorInvocation) {
+ if (source.getDeploymentTargetNodeId().equals(target.getDeploymentTargetNodeId())) {
+ System.out.println("same node - no relay");
+ return true;
+ }
+ return false;
+ }
+ return false;
}
private ElementStatusInfoSettings makeStatusInfoSettings(String elementIdentifier) {
@@ -165,13 +237,11 @@ public class InvocationGraphBuilder {
}
private Set<InvocableStreamPipesEntity> getConnections(NamedStreamPipesEntity source) {
- Set<String> outgoingEdges = pipelineGraph.outgoingEdgesOf(source);
- return outgoingEdges
+ return pipelineGraph.outgoingEdgesOf(source)
.stream()
.map(o -> pipelineGraph.getEdgeTarget(o))
.map(g -> (InvocableStreamPipesEntity) g)
.collect(Collectors.toSet());
-
}
private Integer getIndex(String sourceDomId, InvocableStreamPipesEntity targetElement) {
@@ -191,5 +261,4 @@ public class InvocationGraphBuilder {
.findFirst()
.get();
}
-
}
\ No newline at end of file
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java
index ec3a891..f38212e 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java
@@ -34,14 +34,15 @@ import java.util.Set;
public class ProtocolSelector extends GroundingSelector {
- private String outputTopic;
- private List<SpProtocol> prioritizedProtocols;
+ private final String outputTopic;
+ private final List<SpProtocol> prioritizedProtocols;
public ProtocolSelector(NamedStreamPipesEntity source, Set<InvocableStreamPipesEntity> targets) {
super(source, targets);
this.outputTopic = TopicGenerator.generateRandomTopic();
- this.prioritizedProtocols =
- BackendConfig.INSTANCE.getMessagingSettings().getPrioritizedProtocols();
+ this.prioritizedProtocols = BackendConfig.INSTANCE
+ .getMessagingSettings()
+ .getPrioritizedProtocols();
}
public TransportProtocol getPreferredProtocol() {
@@ -50,36 +51,34 @@ public class ProtocolSelector extends GroundingSelector {
.getEventGrounding()
.getTransportProtocol();
} else {
- for(SpProtocol prioritizedProtocol: prioritizedProtocols) {
- if (prioritizedProtocol.getProtocolClass().equals(KafkaTransportProtocol.class.getCanonicalName()) &&
- supportsProtocol(KafkaTransportProtocol.class)) {
- return kafkaTopic();
- }
- else if (prioritizedProtocol.getProtocolClass().equals(JmsTransportProtocol.class.getCanonicalName()) &&
- supportsProtocol(JmsTransportProtocol.class)) {
- return jmsTopic();
- } else if (prioritizedProtocol.getProtocolClass().equals(MqttTransportProtocol.class.getCanonicalName()) &&
- supportsProtocol(MqttTransportProtocol.class)) {
- return mqttTopic();
+ for(SpProtocol p: prioritizedProtocols) {
+ if (matches(p, KafkaTransportProtocol.class) && supportsProtocol(KafkaTransportProtocol.class)) {
+ return kafkaTransportProtocol();
+ } else if (matches(p, JmsTransportProtocol.class) && supportsProtocol(JmsTransportProtocol.class)) {
+ return jmsTransportProtocol();
+ } else if (matches(p, MqttTransportProtocol.class) && supportsProtocol(MqttTransportProtocol.class)) {
+ return mqttTransportProtocol();
+ } else {
+ throw new IllegalArgumentException("Transport protocol not found: " + p.getProtocolClass());
}
}
}
- return kafkaTopic();
+ throw new IllegalArgumentException("Could not get preferred transport protocol");
}
- private TransportProtocol mqttTopic() {
+ private TransportProtocol mqttTransportProtocol() {
return new MqttTransportProtocol(BackendConfig.INSTANCE.getMqttHost(),
BackendConfig.INSTANCE.getMqttPort(),
outputTopic);
}
- private TransportProtocol jmsTopic() {
+ private TransportProtocol jmsTransportProtocol() {
return new JmsTransportProtocol(BackendConfig.INSTANCE.getJmsHost(),
BackendConfig.INSTANCE.getJmsPort(),
outputTopic);
}
- private TransportProtocol kafkaTopic() {
+ private TransportProtocol kafkaTransportProtocol() {
return new KafkaTransportProtocol(BackendConfig.INSTANCE.getKafkaHost(),
BackendConfig.INSTANCE.getKafkaPort(),
outputTopic,
@@ -87,10 +86,12 @@ public class ProtocolSelector extends GroundingSelector {
BackendConfig.INSTANCE.getZookeeperPort());
}
+ private <T extends TransportProtocol> boolean matches(SpProtocol p, Class<T> clazz) {
+ return p.getProtocolClass().equals(clazz.getCanonicalName());
+ }
- public <T extends TransportProtocol> boolean supportsProtocol(Class<T> protocol) {
+ private <T extends TransportProtocol> boolean supportsProtocol(Class<T> protocol) {
List<InvocableStreamPipesEntity> elements = buildInvocables();
-
return elements
.stream()
.allMatch(e -> e
@@ -98,6 +99,5 @@ public class ProtocolSelector extends GroundingSelector {
.getTransportProtocols()
.stream()
.anyMatch(protocol::isInstance));
-
}
}
diff --git a/streampipes-vocabulary/src/main/java/org/apache/streampipes/vocabulary/StreamPipes.java b/streampipes-vocabulary/src/main/java/org/apache/streampipes/vocabulary/StreamPipes.java
index 9a8d673..8897ffa 100644
--- a/streampipes-vocabulary/src/main/java/org/apache/streampipes/vocabulary/StreamPipes.java
+++ b/streampipes-vocabulary/src/main/java/org/apache/streampipes/vocabulary/StreamPipes.java
@@ -418,4 +418,5 @@ public class StreamPipes {
public static final String PE_CONFIGURED = NS + "isPeConfigured" ;
public static final String HAS_REQUIRED_FILETYPES = NS + "hasRequiredFiletypes" ;
+ public static final String HAS_EVENT_STREAM_RELAYS = NS + "hasEventStreamRelays";
}
diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.html b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.html
index f8c2199..fb900b7 100644
--- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.html
+++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.html
@@ -49,7 +49,18 @@
<mat-slide-toggle color="primary" [(ngModel)]="advancedSettings">
Choose deployment options
</mat-slide-toggle>
- <mat-divider style="margin: 2em 0 2em 0;"></mat-divider>
+ <mat-divider *ngIf="advancedSettings" style="margin: 2em 0 2em 0;"></mat-divider>
+<!-- <div *ngIf="advancedSettings">-->
+<!-- <div fxFlex="100" fxLayout="row">-->
+<!-- <div fxFlex="50" fxLayout="row" fxLayoutAlign="center center">-->
+<!-- Event strategy-->
+<!-- </div>-->
+<!-- </div>-->
+<!-- <div fxFlex="50" fxLayout="row" fxLayoutAlign="start center">-->
+<!-- <mat-slider></mat-slider>-->
+<!-- </div>-->
+<!-- </div>-->
+ <mat-divider *ngIf="advancedSettings" style="margin: 2em 0 2em 0;"></mat-divider>
<div *ngIf="advancedSettings">
<b>Node selection</b>
<div *ngFor="let processors of pipeline.sepas">
diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.scss b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.scss
index c5dce8d..2f8c6bb 100644
--- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.scss
+++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.scss
@@ -43,4 +43,8 @@
.status-subtext {
font-size: 12pt;
+}
+
+mat-slider {
+ width: 300px;
}
\ No newline at end of file
diff --git a/ui/src/app/editor/editor.module.ts b/ui/src/app/editor/editor.module.ts
index db54354..d44fc86 100644
--- a/ui/src/app/editor/editor.module.ts
+++ b/ui/src/app/editor/editor.module.ts
@@ -59,6 +59,7 @@ import {CustomOutputStrategyComponent} from "./components/output-strategy/custom
import {PropertySelectionComponent} from "./components/output-strategy/property-selection/property-selection.component";
import {UserDefinedOutputStrategyComponent} from "./components/output-strategy/user-defined-output/user-defined-output.component";
import {ConnectModule} from "../connect/connect.module";
+import {MatSliderModule} from "@angular/material/slider";
@NgModule({
imports: [
@@ -75,7 +76,8 @@ import {ConnectModule} from "../connect/connect.module";
FormsModule,
MatProgressSpinnerModule,
ShowdownModule,
- ReactiveFormsModule
+ ReactiveFormsModule,
+ MatSliderModule
],
declarations: [
CompatibleElementsComponent,