You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2023/10/12 08:12:56 UTC
[streampipes] branch 1960-better-support-modification-of-existing-adapters updated (bf37e88bb -> 7aa0bfdb9)
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a change to branch 1960-better-support-modification-of-existing-adapters
in repository https://gitbox.apache.org/repos/asf/streampipes.git
from bf37e88bb Add license header
add c436d172c Bump tslib from 2.5.0 to 2.6.2 in /ui (#1963)
add 61e7c0e6d Bump com.google.guava:guava from 32.0.1-jre to 32.1.2-jre (#1962)
add d71af0ecb Bump pyupgrade from 3.12.0 to 3.13.0 in /streampipes-client-python (#1961)
add 1e3bfd988 fix(#1956): Check Pulsar URL in consul migration runner (#1957)
add 5daaa8202 feat(#1814): Add details dialog to service configuration view (#1959)
add 7e1283236 Bump org.xerial.snappy:snappy-java from 1.1.10.1 to 1.1.10.4 (#1964)
add 9cac93252 feat(#1925): add prometheus endpoint (#1950)
add b3230e808 Bump org.mockito:mockito-core from 5.4.0 to 5.5.0 (#1970)
add 7420b7f66 chore(deps-dev): bump org.testcontainers:testcontainers (#1972)
add e34b1a575 [hotfix] Fix dependency conflict of mail components (#1974)
add ebfc654a1 chore(deps): bump com.rabbitmq:amqp-client from 5.18.0 to 5.19.0 (#1977)
add 714f46232 Add configuration hint for Kafka users. (#1945)
add 9b4f8078a Bump numpydoc from 1.5.0 to 1.6.0 in /streampipes-client-python (#1967)
add 513288680 monthly update of vulnerability report (#1982)
add 9e5fdc3cd [#1879] Inject DisplayRecommendedPipe to NaryMappingProperty (#1981)
add 6cf56a911 chore(deps): bump org.yaml:snakeyaml from 2.1 to 2.2 (#1984)
add 2d67d15b5 Add deployment of Prometheus and Grafana to K8s (#1978)
add 598c13996 chore(deps-dev): bump pyupgrade in /streampipes-client-python (#1987)
add 774753e2a chore(deps): bump org.simplejavamail:simple-java-mail (#1988)
add 59a59a8c8 chore(deps): bump urllib3 in /streampipes-client-python (#1990)
add db47ce64e chore(deps-dev): remove org.ow2.asm:asm (#1989)
add e4eb677b1 chore(deps-dev): bump pillow in /streampipes-client-python (#1995)
add 4829537b8 chore(deps): remove com.google.j2objc:j2objc-annotations (#1997)
add 5ed09fdb3 Improvements to prometheus and Grafana (#1991)
add 7acaf752c chore(deps): bump org.checkerframework:checker-qual (#1996)
add 283d57918 chore(deps): bump io.nats:jnats from 2.16.1 to 2.17.0 (#1999)
add e680d47e0 refactor(#1960): Refactor transformation rules, add schema restore (#1994)
add b51326d1f Set default Kafka auto offset explicitly if it is not provided in configuration (#1992) (#1993)
add b751a1c79 chore(deps): bump org.mockito:mockito-core from 5.5.0 to 5.6.0 (#2006)
add d59fc7f94 chore(deps-dev): bump pyupgrade in /streampipes-client-python (#2007)
add c156c72e0 chore(deps): bump org.simplejavamail:simple-java-mail (#2011)
add ed4dd8320 chore(deps): bump com.nimbusds:nimbus-jose-jwt from 9.35 to 9.36 (#2010)
add 8a565bc64 chore(deps-dev): bump webpack from 5.76.1 to 5.88.2 in /ui (#2009)
add 9fbc9c9e7 fix typo in asset management caption (#2012)
add 7eb9c6f0d chore(deps-dev): bump mypy in /streampipes-client-python (#2013)
add 7aa0bfdb9 Merge branch 'dev' into 1960-better-support-modification-of-existing-adapters
No new revisions were added by this update.
Summary of changes:
VULNERABILITY.md | 6 +-
.../deploy/standalone/kafka/docker-compose.dev.yml | 2 +-
installer/compose/docker-compose.full.yml | 6 +-
installer/compose/docker-compose.yml | 6 +-
installer/k8s/README.md | 42 +-
.../k8s/templates/core/backend-deployment.yaml | 2 +
.../core/extensions-all-iiot-deployment.yaml | 2 +
.../templates/external/kafka/kafka-deployment.yaml | 6 +-
.../templates/external/kafka/kafka-service.yaml | 4 +
.../grafana/grafana-deployment.yml} | 53 +-
.../grafana/grafana-pv.yml} | 23 +-
.../grafana/grafana-pvc.yml} | 21 +-
.../grafana/grafana-service.yml} | 12 +-
.../monitor/prometheus/prometheus-configmap.yaml | 49 +
.../monitor/prometheus/prometheus-deployment.yml | 108 +
.../prometheus/prometheus-pv.yml} | 23 +-
.../prometheus/prometheus-pvc.yml} | 21 +-
.../prometheus/prometheus-service.yml} | 13 +-
.../prometheus/token-pvc.yml} | 21 +-
installer/k8s/values.yaml | 39 +
pom.xml | 57 +-
prometheus-grafana/README.md | 29 +
prometheus-grafana/grafana/dashboards/backend.json | 3005 ++++++++++++++++++++
.../grafana/dashboards/extensions-all-iiot.json | 2442 ++++++++++++++++
prometheus-grafana/prometheus/prometheus.yml | 48 +
...ive-data-from-the-streampipes-data-stream.ipynb | 2 +-
streampipes-client-python/poetry.lock | 195 +-
streampipes-client-python/pyproject.toml | 6 +-
streampipes-commons/pom.xml | 4 +
.../apache/streampipes/commons/constants/Envs.java | 6 +-
.../commons/constants/InstanceIdExtractor.java | 4 +
.../commons/environment/DefaultEnvironment.java | 10 +
.../commons/environment/Environment.java | 5 +
.../prometheus/StreamPipesCollectorRegistry.java | 91 +
.../prometheus/pipelines/PipelineFlowMetrics.java | 47 +
.../prometheus/pipelines/PipelineFlowStats.java | 112 +
.../prometheus/pipelines/PipelinesMetrics.java | 57 +
.../prometheus/pipelines/PipelinesStats.java | 166 ++
.../management/AdapterEventPreviewPipeline.java | 3 +-
streampipes-connect-shared/pom.xml | 10 +
.../shared/AdapterPipelineGeneratorBase.java | 95 +-
....java => SupportsNestedTransformationRule.java} | 45 +-
.../convert/ProvidesConversionResult.java | 11 +-
.../preprocessing/convert/SchemaConverter.java | 67 +
.../convert/ToOriginalSchemaConverter.java | 141 +
.../convert/ToTransformedSchemaConverter.java | 140 +
...a => AdapterTransformationPipelineElement.java} | 22 +-
.../TransformSchemaAdapterPipelineElement.java | 81 -
.../elements/TransformStreamAdapterElement.java | 71 -
.../TransformValueAdapterPipelineElement.java | 96 -
...StatefulTransformationRuleGeneratorVisitor.java | 99 +
...tatelessTransformationRuleGeneratorVisitor.java | 134 +
.../TransformationRuleGeneratorVisitor.java} | 18 +-
.../transform/TransformationRule.java | 2 +-
...onRule.java => AddValueTransformationRule.java} | 19 +-
.../schema/CreateNestedTransformationRule.java | 31 +-
.../transform/schema/DeleteTransformationRule.java | 29 +-
.../transform/schema/MoveTransformationRule.java | 18 +-
.../transform/schema/RenameTransformationRule.java | 36 +-
.../transform/schema/SchemaEventTransformer.java | 119 -
.../stream/DuplicateFilterPipelineElement.java | 8 +-
.../stream/EventRateTransformationRule.java | 10 +-
.../transform/stream/StreamEventTransformer.java | 66 -
...le.java => AddTimestampTransformationRule.java} | 16 +-
.../value/CorrectionValueTransformationRule.java | 68 +-
.../value/DatatypeTransformationRule.java | 18 +-
.../value/TimestampTranformationRuleMode.java | 13 +-
...nRule.java => TimestampTransformationRule.java} | 60 +-
.../transform/value/UnitTransformationRule.java | 59 +-
.../transform/value/ValueEventTransformer.java | 83 -
.../preprocessing/utils/ConversionUtils.java | 93 +
.../preprocessing/{Util.java => utils/Utils.java} | 16 +-
.../connect/shared/preprocessing/UtilsTest.java} | 37 +-
.../shared/preprocessing/convert/Helpers.java | 92 +
.../convert/ToOriginalSchemaConverterTest.java | 104 +
.../convert/ToTransformedSchemaConverterTest.java | 121 +
.../AdapterTransformationPipelineElementTest.java | 60 +
.../schema/CreateNestedTransformationRuleTest.java | 4 +-
.../schema/DeleteTransformationRuleTest.java | 4 +-
.../schema/MoveTransformationRuleTest.java | 6 +-
.../schema/RenameTransformationRuleTest.java | 4 +-
.../schema/SchemaEventTransformerTest.java | 9 +-
.../transform/value/CorrectionValueTest.java | 14 +-
.../transform/value/UnitTransformRuleTest.java | 10 +-
.../transform/value/ValueEventTransformerTest.java | 10 +-
streampipes-extensions-management/pom.xml | 5 -
.../connect/adapter/AdapterPipelineGenerator.java | 32 +-
.../generic/elements/AddTimestampTest.java | 6 +-
.../generic/elements/DuplicateFilterTest.java | 34 +-
.../streampipes-extensions-all-jvm/Dockerfile | 2 +-
.../pom.xml | 6 -
.../config/kafka/kafka/KafkaConnectUtils.java | 15 +-
.../streampipes-sinks-brokers-jvm/pom.xml | 4 -
streampipes-integration-tests/pom.xml | 2 +-
...iption.java => ITransformationRuleVisitor.java} | 53 +-
.../rules/TransformationRuleDescription.java | 4 +
.../connect/rules/TransformationRulePriority.java | 31 +-
.../rules/schema/CreateNestedRuleDescription.java | 13 +
.../rules/schema/DeleteRuleDescription.java | 13 +
.../connect/rules/schema/MoveRuleDescription.java | 13 +
.../rules/schema/RenameRuleDescription.java | 13 +
.../EventRateTransformationRuleDescription.java | 13 +
...oveDuplicatesTransformationRuleDescription.java | 13 +
.../rules/value/AddTimestampRuleDescription.java | 13 +
.../AddValueTransformationRuleDescription.java | 13 +
...hangeDatatypeTransformationRuleDescription.java | 13 +
...rrectionValueTransformationRuleDescription.java | 13 +
.../TimestampTranfsformationRuleDescription.java | 13 +
.../rules/value/UnitTransformRuleDescription.java | 13 +
.../manager/health/PipelineHealthCheck.java | 46 +-
.../monitoring/pipeline/ExtensionsLogProvider.java | 4 +
.../pipeline/ExtensionsServiceLogExecutor.java | 34 +
streampipes-rest/pom.xml | 6 -
.../streampipes/rest/impl/EmailResource.java | 8 +-
streampipes-service-base/pom.xml | 8 +
.../service/base/ServiceBaseConfig.java | 51 +
.../service/base/StreamPipesServiceBase.java | 9 +-
.../service/core/StreamPipesCoreApplication.java | 3 +-
.../service/core/StreamPipesPrometheusConfig.java | 19 +-
.../migrations/v093/ConsulConfigMigration.java | 13 +-
.../streampipes/svcdiscovery/api/SpConfig.java | 2 +
.../svcdiscovery/consul/ConsulSpConfig.java | 7 +-
ui/package-lock.json | 1101 +++++--
ui/package.json | 4 +-
.../manage-asset-links-dialog.component.html | 2 +-
ui/src/app/configuration/configuration.module.ts | 4 +-
...xtensions-service-details-dialog.component.html | 69 +
...xtensions-service-details-dialog.component.scss | 24 +-
.../extensions-service-details-dialog.component.ts | 43 +-
.../email-configuration.component.html | 6 +-
.../email-configuration.component.scss | 8 +
.../email-configuration.component.ts | 2 +-
...extensions-service-configuration.component.html | 32 +-
.../extensions-service-configuration.component.ts | 1 -
.../service-configs/service-configs.component.html | 1 +
.../service-configs/service-configs.component.ts | 2 +-
.../registered-extensions-services.component.html | 61 +-
.../registered-extensions-services.component.scss | 0
.../registered-extensions-services.component.ts | 20 +-
.../event-property-row.component.ts | 25 +-
.../event-schema/event-schema.component.ts | 1 +
.../edit-schema-transformation.component.html | 2 +-
.../edit-event-property.component.html | 1 -
.../static-mapping-nary.component.ts | 17 +-
144 files changed, 9413 insertions(+), 1590 deletions(-)
copy installer/k8s/templates/{core/extensions-all-iiot-deployment.yaml => monitor/grafana/grafana-deployment.yml} (56%)
copy installer/k8s/templates/{external/kafka/kafka-service.yaml => monitor/grafana/grafana-pv.yml} (62%)
copy installer/k8s/templates/{external/kafka/kafka-service.yaml => monitor/grafana/grafana-pvc.yml} (65%)
copy installer/k8s/templates/{external/kafka/kafka-service.yaml => monitor/grafana/grafana-service.yml} (73%)
create mode 100644 installer/k8s/templates/monitor/prometheus/prometheus-configmap.yaml
create mode 100644 installer/k8s/templates/monitor/prometheus/prometheus-deployment.yml
copy installer/k8s/templates/{external/kafka/kafka-service.yaml => monitor/prometheus/prometheus-pv.yml} (61%)
copy installer/k8s/templates/{external/kafka/kafka-service.yaml => monitor/prometheus/prometheus-pvc.yml} (64%)
copy installer/k8s/templates/{external/kafka/kafka-service.yaml => monitor/prometheus/prometheus-service.yml} (73%)
copy installer/k8s/templates/{external/kafka/kafka-service.yaml => monitor/prometheus/token-pvc.yml} (67%)
create mode 100644 prometheus-grafana/README.md
create mode 100644 prometheus-grafana/grafana/dashboards/backend.json
create mode 100644 prometheus-grafana/grafana/dashboards/extensions-all-iiot.json
create mode 100644 prometheus-grafana/prometheus/prometheus.yml
create mode 100644 streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java
create mode 100644 streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/pipelines/PipelineFlowMetrics.java
create mode 100644 streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/pipelines/PipelineFlowStats.java
create mode 100644 streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/pipelines/PipelinesMetrics.java
create mode 100644 streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/pipelines/PipelinesStats.java
copy streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/{transform/schema/RenameTransformationRule.java => SupportsNestedTransformationRule.java} (51%)
rename streampipes-extensions/streampipes-connect-adapters/src/test/java/org/apache/streampipes/connect/adapters/generic/Mock.java => streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ProvidesConversionResult.java (69%)
create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/SchemaConverter.java
create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverter.java
create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java
rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/{AddTimestampPipelineElement.java => AdapterTransformationPipelineElement.java} (52%)
delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformSchemaAdapterPipelineElement.java
delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformStreamAdapterElement.java
delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/TransformValueAdapterPipelineElement.java
create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java
create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatelessTransformationRuleGeneratorVisitor.java
rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/{transform/stream/StreamTransformationRule.java => generator/TransformationRuleGeneratorVisitor.java} (63%)
rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/{SchemaTransformationRule.java => AddValueTransformationRule.java} (69%)
delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformer.java
delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/StreamEventTransformer.java
rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/{ValueTransformationRule.java => AddTimestampTransformationRule.java} (71%)
rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/{TimestampTranformationRule.java => TimestampTransformationRule.java} (53%)
delete mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformer.java
create mode 100644 streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/ConversionUtils.java
copy streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/{Util.java => utils/Utils.java} (71%)
rename streampipes-connect-shared/src/{main/java/org/apache/streampipes/connect/shared/preprocessing/Util.java => test/java/org/apache/streampipes/connect/shared/preprocessing/UtilsTest.java} (52%)
create mode 100644 streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/Helpers.java
create mode 100644 streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverterTest.java
create mode 100644 streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverterTest.java
create mode 100644 streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElementTest.java
copy streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/{TransformationRuleDescription.java => ITransformationRuleVisitor.java} (62%)
copy streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/InstanceIdExtractor.java => streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRulePriority.java (62%)
create mode 100644 streampipes-service-base/src/main/java/org/apache/streampipes/service/base/ServiceBaseConfig.java
copy streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/InstanceIdExtractor.java => streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesPrometheusConfig.java (63%)
create mode 100644 ui/src/app/configuration/dialog/extensions-service-details/extensions-service-details-dialog.component.html
rename streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/TimestampTransformationRuleMode.java => ui/src/app/configuration/dialog/extensions-service-details/extensions-service-details-dialog.component.scss (70%)
rename streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AddValuePipelineElement.java => ui/src/app/configuration/dialog/extensions-service-details/extensions-service-details-dialog.component.ts (55%)
rename ui/src/app/configuration/extensions-service-management/{registrered-extensions-services => registered-extensions-services}/registered-extensions-services.component.html (61%)
rename ui/src/app/configuration/extensions-service-management/{registrered-extensions-services => registered-extensions-services}/registered-extensions-services.component.scss (100%)
rename ui/src/app/configuration/extensions-service-management/{registrered-extensions-services => registered-extensions-services}/registered-extensions-services.component.ts (70%)