You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2023/08/04 18:00:27 UTC
[airflow] 01/01: openlineage: move config to provider.yaml
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch move-ol-config-to-local-file
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 5e7dcc1a172d0c5659bb34622ece1d02a24a7104
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Fri Aug 4 19:49:33 2023 +0200
openlineage: move config to provider.yaml
Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
airflow/config_templates/config.yml | 59 ----------------------------
airflow/providers/openlineage/provider.yaml | 61 +++++++++++++++++++++++++++++
2 files changed, 61 insertions(+), 59 deletions(-)
diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml
index 33c3ffeb1d..e5bce00987 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -1230,65 +1230,6 @@ lineage:
type: string
example: ~
default: ""
-openlineage:
- description: |
- This section applies settings for OpenLineage integration.
- For backwards compatibility with `openlineage-python` one can still use
- `openlineage.yml` file or `OPENLINEAGE_` environment variables. However, below
- configuration takes precedence over those.
- More in documentation - https://openlineage.io/docs/client/python#configuration.
- options:
- disabled:
- description: |
- Set this to true if you don't want OpenLineage to emit events.
- type: boolean
- example: ~
- default: "False"
- version_added: ~
- namespace:
- description: |
- OpenLineage namespace
- version_added: ~
- type: string
- example: "food_delivery"
- default: ~
- extractors:
- description: |
- Comma-separated paths to custom OpenLineage extractors.
- type: string
- example: full.path.to.ExtractorClass;full.path.to.AnotherExtractorClass
- default: ''
- version_added: ~
- config_path:
- description: |
- Path to YAML config. This provides backwards compatibility to pass config as
- `openlineage.yml` file.
- version_added: ~
- type: string
- example: ~
- default: ''
- transport:
- description: |
- OpenLineage Client transport configuration. It should contain type
- and additional options per each type.
-
- Currently supported types are:
-
- * HTTP
- * Kafka
- * Console
- type: string
- example: '{"type": "http", "url": "http://localhost:5000"}'
- default: ''
- version_added: ~
- disable_source_code:
- description: |
- If disabled, OpenLineage events do not contain source code of particular
- operators, like PythonOperator.
- default: ~
- example: ~
- type: boolean
- version_added: ~
operators:
description: ~
options:
diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml
index 2a2c1a6669..5c64c9f6c3 100644
--- a/airflow/providers/openlineage/provider.yaml
+++ b/airflow/providers/openlineage/provider.yaml
@@ -41,3 +41,64 @@ integrations:
plugins:
- name: openlineage
plugin-class: airflow.providers.openlineage.plugins.openlineage.OpenLineageProviderPlugin
+
+config:
+ openlineage:
+ description: |
+ This section applies settings for OpenLineage integration.
+ For backwards compatibility with `openlineage-python` one can still use
+ `openlineage.yml` file or `OPENLINEAGE_` environment variables. However, below
+ configuration takes precedence over those.
+ More in documentation - https://openlineage.io/docs/client/python#configuration.
+ options:
+ disabled:
+ description: |
+ Set this to true if you don't want OpenLineage to emit events.
+ type: boolean
+ example: ~
+ default: "False"
+ version_added: ~
+ namespace:
+ description: |
+ OpenLineage namespace
+ version_added: ~
+ type: string
+ example: "food_delivery"
+ default: ~
+ extractors:
+ description: |
+ Comma-separated paths to custom OpenLineage extractors.
+ type: string
+ example: full.path.to.ExtractorClass;full.path.to.AnotherExtractorClass
+ default: ''
+ version_added: ~
+ config_path:
+ description: |
+ Path to YAML config. This provides backwards compatibility to pass config as
+ `openlineage.yml` file.
+ version_added: ~
+ type: string
+ example: ~
+ default: ''
+ transport:
+ description: |
+ OpenLineage Client transport configuration. It should contain type
+ and additional options per each type.
+
+ Currently supported types are:
+
+ * HTTP
+ * Kafka
+ * Console
+ type: string
+ example: '{"type": "http", "url": "http://localhost:5000"}'
+ default: ''
+ version_added: ~
+ disable_source_code:
+ description: |
+ If disabled, OpenLineage events do not contain source code of particular
+ operators, like PythonOperator.
+ default: ~
+ example: ~
+ type: boolean
+ version_added: ~