You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2023/02/23 11:42:17 UTC
[streampipes] 01/02: Merge branch 'dev' into 1207-improve-helm-charts
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch 1207-improve-helm-charts
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 571f230ce4f562fc92ce6bc1463468a77a349fd9
Merge: c6f79e2eb d70630c2d
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Thu Feb 23 11:25:04 2023 +0100
Merge branch 'dev' into 1207-improve-helm-charts
.asf.yaml | 10 +-
.github/dependabot.yml | 3 +
.github/pr-labeler-config.yml | 3 +
.github/workflows/build.yml | 6 +-
.github/workflows/cypress-test.yml | 4 +-
.github/workflows/pr-labeler.yml | 14 +-
.github/workflows/pr-validation.yml | 20 +-
.../{pr-labeler.yml => pypi-deployment.yml} | 47 +-
.github/workflows/python-docs.yml | 59 +++
README.md | 3 +-
.../streampipes-archetype-extensions-jvm/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 2 +-
.../pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 2 +-
.../streampipes-archetype-pe-sinks-flink/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 2 +-
installer/cli/.env | 6 +-
installer/cli/README.md | 2 +-
.../deploy/standalone/epsg/docker-compose.dev.yml | 8 +-
.../cli/deploy/standalone/epsg/docker-compose.yml | 44 ++
installer/cli/environments/backend | 3 +-
installer/compose/.env | 2 +-
installer/compose/README.md | 2 +-
installer/k8s/Chart.yaml | 4 +-
installer/k8s/README.md | 2 +-
installer/k8s/values.yaml | 2 +-
installer/scripts/epsg/EPSG_FINISH.sql | 17 +
installer/scripts/epsg/PostgreSQL_Data_Script.sql | 17 +
installer/scripts/epsg/PostgreSQL_FKey_Script.sql | 17 +
installer/scripts/epsg/PostgreSQL_Table_Script.sql | 17 +
installer/scripts/epsg/Readme.md | 140 +++++
pom.xml | 61 +--
streampipes-client-python/.pre-commit-config.yaml | 2 +-
streampipes-client-python/Makefile | 14 +-
streampipes-client-python/README.md | 36 +-
...introduction-to-streampipes-python-client.ipynb | 8 +-
...cting-data-from-the-streampipes-data-lake.ipynb | 12 +-
...ive-data-from-the-streampipes-data-stream.ipynb | 70 ++-
...ine-learning-on-a-streampipes-data-stream.ipynb | 484 +++++++++++++++++
.../docs/getting-started/developing.md | 106 ++++
.../docs/getting-started/first-steps.md | 68 +++
.../docs/getting-started/installation.md | 41 --
.../docs/getting-started/quickstart.md | 22 +-
streampipes-client-python/docs/index.md | 39 +-
.../docs/scripts/gen_ref_pages.py | 8 +-
streampipes-client-python/mkdocs.yml | 7 +-
streampipes-client-python/setup.py | 8 +-
.../__init__.py | 0
.../__version__.py | 2 +-
.../client/__init__.py | 0
.../client/client.py | 12 +-
.../client/config.py} | 2 +-
.../client/credential_provider.py | 0
.../endpoint/__init__.py | 0
.../endpoint/api/__init__.py | 0
.../endpoint/api/data_lake_measure.py | 14 +-
.../endpoint/api/data_stream.py | 12 +-
.../endpoint/endpoint.py | 36 +-
.../endpoint/exceptions.py | 0
.../endpoint/messaging/__init__.py | 0
.../function_zoo}/__init__.py | 0
.../streampipes/function_zoo/river_function.py | 184 +++++++
.../utils => streampipes/functions}/__init__.py | 0
.../functions/broker/__init__.py | 5 +-
.../functions/broker/broker.py | 55 +-
.../streampipes/functions/broker/broker_handler.py | 53 ++
.../functions/broker/nats_broker.py | 31 +-
.../functions/broker/output_collector.py | 65 +++
.../functions/function_handler.py | 57 +-
.../functions/registration.py | 2 +-
.../functions/streampipes_function.py | 39 +-
.../functions/utils}/__init__.py | 0
.../functions/utils/async_iter_handler.py | 0
.../functions/utils/data_stream_context.py | 6 +-
.../functions/utils/data_stream_generator.py | 73 +++
.../functions/utils/function_context.py | 4 +-
.../model}/__init__.py | 0
.../model/common.py | 68 +--
.../model/container/__init__.py | 0
.../model/container/data_lake_measures.py | 6 +-
.../model/container/data_streams.py | 6 +-
.../model/container/resource_container.py | 2 +-
.../model/resource/__init__.py | 0
.../model/resource/data_lake_measure.py | 4 +-
.../model/resource/data_lake_series.py | 2 +-
.../model/resource/data_stream.py | 75 ++-
.../model/resource/function_definition.py | 35 +-
.../model/resource/resource.py | 2 +-
.../tests/client/test_client.py | 16 +-
.../tests/client/test_credential_provider.py | 2 +-
.../tests/client/test_data_lake_series.py | 12 +-
.../tests/client/test_endpoint.py | 81 +--
.../tests/functions/test_function_handler.py | 110 +++-
.../tests/functions/test_river_function.py | 206 ++++++++
.../model}/__init__.py | 0
.../model/resource}/__init__.py | 0
.../tests/model/resource/test_data_stream.py | 73 +++
streampipes-client/pom.xml | 21 +-
.../streampipes/client/api/DataProcessorApi.java | 3 +-
.../apache/streampipes/client/api/DataSinkApi.java | 3 +-
.../streampipes/client/api/DataStreamApi.java | 3 +-
.../apache/streampipes/client/api/PipelineApi.java | 2 +-
streampipes-commons/pom.xml | 2 +-
.../streampipes/commons/constants/CustomEnvs.java | 20 +-
.../commons/constants/DefaultEnvValues.java | 9 +-
.../apache/streampipes/commons/constants/Envs.java | 87 ++-
.../commons/environment/DefaultEnvironment.java | 80 +++
.../commons/environment/Environment.java | 40 ++
.../commons/environment/Environments.java | 16 +-
.../environment/variable/EnvironmentVariable.java | 20 +-
.../streampipes/commons/networking/Networking.java | 21 +-
streampipes-config/pom.xml | 13 +-
.../streampipes/config/backend/BackendConfig.java | 104 +---
.../config/backend/BackendConfigKeys.java | 11 -
streampipes-connect-management/pom.xml | 15 +-
streampipes-data-explorer-commons/pom.xml | 13 +-
streampipes-data-explorer/pom.xml | 13 +-
.../dataexplorer/DataLakeManagementV4.java | 10 +-
.../dataexplorer/query/DataExplorerQuery.java | 10 +-
.../dataexplorer/sdk/DataLakeQueryBuilder.java | 8 +-
.../dataexplorer/v4/query/DataExplorerQueryV4.java | 24 +-
streampipes-data-export/pom.xml | 10 +-
.../export/resolver/AbstractResolver.java | 7 +
.../export/resolver/AdapterResolver.java | 3 +-
.../export/resolver/DataSourceResolver.java | 3 +-
.../export/resolver/PipelineResolver.java | 9 +-
.../export/utils/EventGroundingProcessor.java | 60 ++-
streampipes-dataformat-cbor/pom.xml | 9 +-
streampipes-dataformat-fst/pom.xml | 9 +-
streampipes-dataformat-json/pom.xml | 9 +-
streampipes-dataformat-smile/pom.xml | 9 +-
streampipes-dataformat/pom.xml | 7 +-
streampipes-extensions-api/pom.xml | 9 +-
streampipes-extensions-management/pom.xml | 37 +-
...StreamPipesClientRuntimeConnectionResolver.java | 21 +-
.../management/connect/adapter/Adapter.java | 16 -
.../elements/SendToBrokerAdapterSink.java | 15 +-
.../elements/SendToJmsAdapterSink.java | 4 +-
.../elements/SendToKafkaAdapterSink.java | 6 +-
.../elements/SendToMqttAdapterSink.java | 4 +-
.../elements/SendToNatsAdapterSink.java | 4 +-
streampipes-extensions/.github/workflows/build.yml | 587 ---------------------
streampipes-extensions/pom.xml | 6 +-
.../streampipes-connect-adapters-iiot/pom.xml | 8 +-
.../streampipes-connect-adapters/pom.xml | 6 +-
.../streampipes-connectors-influx/pom.xml | 12 +-
.../streampipes-extensions-all-iiot/pom.xml | 26 +-
.../streampipes-extensions-all-jvm/pom.xml | 12 +-
.../pom.xml | 18 +-
.../streampipes-pipeline-elements-all-jvm/pom.xml | 28 +-
.../pom.xml | 2 +-
.../streampipes-pipeline-elements-shared/pom.xml | 10 +-
.../pom.xml | 8 +-
.../pom.xml | 6 +-
.../streampipes-processors-enricher-flink/pom.xml | 6 +-
.../streampipes-processors-enricher-jvm/pom.xml | 6 +-
.../streampipes-processors-filters-jvm/pom.xml | 6 +-
.../processors/filters/jvm/FiltersJvmInit.java | 4 +-
.../processor/sdt/SwingingDoorTrendingFilter.java | 182 +++++++
.../sdt/SwingingDoorTrendingFilterProcessor.java | 133 +++++
.../documentation.md | 79 +++
.../icon.png | Bin 0 -> 107607 bytes
.../strings.en | 34 ++
.../streampipes-processors-filters-siddhi/pom.xml | 8 +-
.../streampipes-processors-geo-flink/pom.xml | 6 +-
.../streampipes-processors-geo-jvm/pom.xml | 20 +-
.../streampipes/processors/geo/jvm/GeoJvmInit.java | 32 +-
.../SpNotSupportedGeometryException.java | 24 +-
.../geo/jvm/jts/helper/SpGeometryBuilder.java | 60 ++-
.../geo/jvm/jts/helper/SpReprojectionBuilder.java | 304 +++++++++++
.../LatLngToJtsPointProcessor.java | 7 +-
.../reprojection/ReprojectionProcessor.java | 146 +++++
.../trajectory/TrajectoryFromPointsProcessor.java | 16 +-
.../HaversineDistanceCalculatorProcessor.java | 21 +-
.../speedcalculator/SpeedCalculatorProcessor.java | 6 +-
.../documentation.md | 56 +-
.../icon.png | Bin 0 -> 20720 bytes
.../strings.en | 14 +-
.../pom.xml | 12 +-
.../pom.xml | 29 +-
.../pom.xml | 6 +-
.../pom.xml | 8 +-
.../streampipes-processors-text-mining-jvm/pom.xml | 11 +-
.../pom.xml | 10 +-
.../pom.xml | 8 +-
.../streampipes-sinks-brokers-jvm/pom.xml | 8 +-
.../streampipes-sinks-databases-flink/pom.xml | 27 +-
.../streampipes-sinks-databases-jvm/pom.xml | 6 +-
.../streampipes-sinks-internal-jvm/pom.xml | 10 +-
.../streampipes-sinks-notifications-jvm/pom.xml | 8 +-
.../streampipes-sources-vehicle-simulator/pom.xml | 10 +-
.../simulator/simulator/VehicleDataSimulator.java | 13 +-
.../pom.xml | 13 +-
.../simulator/utils/WatertankDataSimulator.java | 13 +-
streampipes-integration-tests/pom.xml | 8 +-
streampipes-logging/pom.xml | 5 +-
streampipes-mail/pom.xml | 9 +-
streampipes-maven-plugin/pom.xml | 9 +-
streampipes-measurement-units/pom.xml | 7 +-
streampipes-messaging-jms/pom.xml | 4 +-
streampipes-messaging-kafka/pom.xml | 6 +-
.../messaging/kafka/SpKafkaProducer.java | 6 +-
streampipes-messaging-mqtt/pom.xml | 4 +-
streampipes-messaging-nats/pom.xml | 4 +-
streampipes-messaging/pom.xml | 4 +-
streampipes-model-client/pom.xml | 9 +-
streampipes-model-shared/pom.xml | 2 +-
streampipes-model/pom.xml | 13 +-
.../streampipes/model/datalake/DataSeries.java | 15 +
.../{DataSeries.java => DataSeriesBuilder.java} | 52 +-
.../model/datalake/SpQueryResultBuilder.java | 73 +++
.../model/datalake/DataSeriesBuilderTest.java | 90 ++++
.../model/datalake/SpQueryResultBuilderTest.java | 116 ++++
streampipes-pipeline-management/pom.xml | 42 +-
.../provider/StoredPipelineElementProvider.java | 13 +-
.../runtime/PipelineElementRuntimeInfoFetcher.java | 13 +-
.../manager/setup/AutoInstallation.java | 56 +-
streampipes-platform-services/pom.xml | 15 +-
streampipes-resource-management/pom.xml | 11 +-
.../AbstractPipelineElementResourceManager.java | 9 +-
.../resource/management/UserResourceManager.java | 11 +-
streampipes-rest-core-base/pom.xml | 13 +-
streampipes-rest-extensions/pom.xml | 10 +-
.../pe/InvocablePipelineElementResource.java | 4 +-
streampipes-rest-shared/pom.xml | 9 +-
streampipes-rest/pom.xml | 42 +-
.../streampipes/rest/impl/PipelineResource.java | 37 --
.../org/apache/streampipes/rest/impl/Setup.java | 3 +
.../org/apache/streampipes/rest/impl/Version.java | 3 +
.../rest/impl/pe/DataProcessorResource.java | 12 +-
.../streampipes/rest/impl/pe/DataSinkResource.java | 12 +-
.../rest/impl/pe/DataStreamResource.java | 31 +-
streampipes-sdk-bundle/pom.xml | 25 +-
streampipes-sdk/pom.xml | 11 +-
...AbstractConfigurablePipelineElementBuilder.java | 18 +
streampipes-security-jwt/pom.xml | 7 +-
.../streampipes/security/jwt/KeyGenerator.java | 5 +-
streampipes-serializers-json/pom.xml | 11 +-
streampipes-service-base/pom.xml | 6 +-
.../resources/{logback-spring.xml => logback.xml} | 0
streampipes-service-core/pom.xml | 17 +-
.../service/core/StreamPipesEnvChecker.java | 40 +-
.../src/main/resources/application.properties | 2 +-
streampipes-service-discovery-api/pom.xml | 2 +-
streampipes-service-discovery-consul/pom.xml | 8 +-
.../svcdiscovery/consul/ConsulProvider.java | 2 +-
streampipes-service-discovery/pom.xml | 6 +-
streampipes-service-extensions/pom.xml | 6 +-
.../extensions/security/WebSecurityConfig.java | 12 +-
.../src/main/resources/application.properties | 2 +-
streampipes-sources/pom.xml | 7 +-
streampipes-storage-api/pom.xml | 9 +-
streampipes-storage-couchdb/pom.xml | 15 +-
.../storage/couchdb/utils/CouchDbConfig.java | 55 --
.../streampipes/storage/couchdb/utils/Utils.java | 27 +-
streampipes-storage-management/pom.xml | 9 +-
streampipes-test-utils/pom.xml | 7 +-
streampipes-user-management/pom.xml | 13 +-
.../encryption/SecretEncryptionManager.java | 7 +-
.../user/management/jwt/JwtTokenProvider.java | 7 +-
streampipes-vocabulary/pom.xml | 5 +-
streampipes-wrapper-distributed/pom.xml | 13 +-
streampipes-wrapper-flink/pom.xml | 7 +-
streampipes-wrapper-kafka-streams/pom.xml | 7 +-
streampipes-wrapper-python/pom.xml | 5 +-
streampipes-wrapper-siddhi/pom.xml | 4 +-
streampipes-wrapper-standalone/pom.xml | 7 +-
.../standalone/function/StreamPipesFunction.java | 10 +-
streampipes-wrapper/pom.xml | 17 +-
ui/README.md | 11 +-
ui/package-lock.json | 343 ++++--------
ui/package.json | 14 +-
.../src/lib/apis/pipeline-element.service.ts | 6 +-
.../src/lib/apis/pipeline.service.ts | 12 +-
.../assets/dialog/base-asset-links.directive.ts | 2 +-
ui/src/app/home/components/status.component.ts | 2 +-
ui/src/app/info/about/about.component.html | 23 +
.../app/notifications/notifications.component.ts | 2 +-
ui/src/app/pipelines/pipelines.component.ts | 2 +-
.../token/token-management-settings.component.html | 23 +-
.../token/token-management-settings.component.scss | 4 +
.../token/token-management-settings.component.ts | 8 +
282 files changed, 4921 insertions(+), 2416 deletions(-)
diff --cc installer/k8s/values.yaml
index 81fa886e3,07edd3dbf..7a71e2733
--- a/installer/k8s/values.yaml
+++ b/installer/k8s/values.yaml
@@@ -21,67 -21,30 +21,67 @@@ persistentVolumeReclaimPolicy: "Delete
persistentVolumeAccessModes: "ReadWriteOnce"
streampipes:
- version: "0.91.0-SNAPSHOT"
+ version: "0.92.0-SNAPSHOT"
registry: "apachestreampipes"
+ core:
+ appName: "backend"
+ port: 8030
+ persistence:
+ storageClassName: ""
+ claimName: "backend-pvc"
+ pvName: "backend-pv"
+ service:
+ name: "backend"
+ port: 8030
+ ui:
+ appName: "ui"
+ service:
+ type: "LoadBalancer"
+ nodePort: 8088
+ port: 8088
+ port: 8088
external:
- consulVersion: 1.14.3
- couchdbVersion: 3.3.1
- flinkVersion: 1.13.5-scala_2.11
+ consul:
+ appName: "consul"
+ version: 1.14.3
+ persistence:
+ storageClassName: ""
+ claimName: "consul-pvc"
+ pvName: "consul-pv"
+ couchdb:
+ appName: "couchdb"
+ version: 3.3.1
+ user: "admin"
+ password: "admin"
+ persistence:
+ storageClassName: ""
+ claimName: "couchdb-pvc"
+ pvName: "couchdb-pv"
+ influxdb:
+ appName: "influxdb"
+ version: 2.6
+ username: "admin"
+ password: "sp-admin"
+ adminToken: "sp-admin"
+ initOrg: "sp"
+ initBucket: "sp"
+ # For database migration in v0.91.0 - set init mode to 'upgrade' to migrate an existing installation
+ initMode: "setup"
+ persistence:
+ storageClassName: ""
+ claimName: "influxdb2-pvc"
+ claimNameV1: "influxdb-pvc"
+ pvName: "influxdb2-pv"
+ pvNameV1: "influxdb-pv"
+ nats:
+ appName: "nats"
+ service:
+ type: "LoadBalancer"
+ externalTrafficPolicy: "Local"
+ version:
kafkaVersion: 2.2.0
zookeeperVersion: 3.4.13
- influxdbVersion: 2.6
-# For database migration in v0.91.0 - set init mode to 'upgrade' to migrate an existing installation
-influxdb:
- username: "admin"
- password: "sp-admin"
- adminToken: "sp-admin"
- initOrg: "sp"
- initBucket: "sp"
- initMode: "setup"
-
-
-# NOTE: Local testing only!
-# minikube start --mount-string ${HOME}/streampipes-k8s:/streampipes-k8s --mount --memory=4g --cpus=4
-# Specify minikube directory:
-hostPath: "/streampipes-k8s"