(airflow) 01/01: openlineage: add file dataset type support into common.io provider

[mo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

mobuchowski pushed a commit to branch aip-62/common-io-file
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 70b653f78ca68c7b72d1b9eb1840e66e0ce0b966
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Tue Jul 16 01:50:21 2024 +0200

    openlineage: add file dataset type support into common.io provider
    
    Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
 .../providers/common/io/datasets/__init__.py                       | 5 -----
 .../test_empty.py => airflow/providers/common/io/datasets/file.py  | 7 +++++--
 airflow/providers/common/io/provider.yaml                          | 5 +++++
 .../common/{compat/test_empty.py => io/datasets/__init__.py}       | 5 -----
 .../common/{compat/test_empty.py => io/datasets/test_file.py}      | 7 +++++--
 5 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/tests/providers/common/compat/test_empty.py b/airflow/providers/common/io/datasets/__init__.py
similarity index 91%
copy from tests/providers/common/compat/test_empty.py
copy to airflow/providers/common/io/datasets/__init__.py
index f91891fb55..13a83393a9 100644
--- a/tests/providers/common/compat/test_empty.py
+++ b/airflow/providers/common/io/datasets/__init__.py
@@ -14,8 +14,3 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from __future__ import annotations
-
-
-def test_empty():
-    assert True
diff --git a/tests/providers/common/compat/test_empty.py b/airflow/providers/common/io/datasets/file.py
similarity index 81%
copy from tests/providers/common/compat/test_empty.py
copy to airflow/providers/common/io/datasets/file.py
index f91891fb55..46c7499037 100644
--- a/tests/providers/common/compat/test_empty.py
+++ b/airflow/providers/common/io/datasets/file.py
@@ -16,6 +16,9 @@
 # under the License.
 from __future__ import annotations
 
+from airflow.datasets import Dataset
 
-def test_empty():
-    assert True
+
+def create_dataset(*, path: str) -> Dataset:
+    # We assume that we get absolute path starting with /
+    return Dataset(uri=f"file://{path}")
diff --git a/airflow/providers/common/io/provider.yaml b/airflow/providers/common/io/provider.yaml
index aa3c2ac901..a45d3d7dfe 100644
--- a/airflow/providers/common/io/provider.yaml
+++ b/airflow/providers/common/io/provider.yaml
@@ -51,6 +51,11 @@ operators:
 xcom:
   - airflow.providers.common.io.xcom.backend
 
+dataset-uris:
+  - schemes: [file]
+    handler: null
+    factory: airflow.providers.common.io.datasets.file.create_dataset
+
 config:
   common.io:
     description: Common IO configuration section
diff --git a/tests/providers/common/compat/test_empty.py b/tests/providers/common/io/datasets/__init__.py
similarity index 91%
copy from tests/providers/common/compat/test_empty.py
copy to tests/providers/common/io/datasets/__init__.py
index f91891fb55..13a83393a9 100644
--- a/tests/providers/common/compat/test_empty.py
+++ b/tests/providers/common/io/datasets/__init__.py
@@ -14,8 +14,3 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from __future__ import annotations
-
-
-def test_empty():
-    assert True
diff --git a/tests/providers/common/compat/test_empty.py b/tests/providers/common/io/datasets/test_file.py
similarity index 79%
rename from tests/providers/common/compat/test_empty.py
rename to tests/providers/common/io/datasets/test_file.py
index f91891fb55..43d63cb205 100644
--- a/tests/providers/common/compat/test_empty.py
+++ b/tests/providers/common/io/datasets/test_file.py
@@ -16,6 +16,9 @@
 # under the License.
 from __future__ import annotations
 
+from airflow.datasets import Dataset
+from airflow.providers.common.io.datasets.file import create_dataset
 
-def test_empty():
-    assert True
+
+def test_file_dataset():
+    assert create_dataset(path="/asdf/fdsa") == Dataset(uri="file:///asdf/fdsa")