You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/11/05 22:24:24 UTC
[airflow] branch master updated: Refactor Elasticsearch provider to
support 1.10.x (#11509)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new cb070e9 Refactor Elasticsearch provider to support 1.10.x (#11509)
cb070e9 is described below
commit cb070e928b73368898a79a6cd41517e680dc3834
Author: Marcus Levine <ma...@gmail.com>
AuthorDate: Thu Nov 5 17:20:36 2020 -0500
Refactor Elasticsearch provider to support 1.10.x (#11509)
---
provider_packages/refactor_provider_packages.py | 65 ++++++++++++++++++++++
.../ci_prepare_provider_readme.sh | 2 +-
2 files changed, 66 insertions(+), 1 deletion(-)
diff --git a/provider_packages/refactor_provider_packages.py b/provider_packages/refactor_provider_packages.py
index ccacef2..8772fa3 100755
--- a/provider_packages/refactor_provider_packages.py
+++ b/provider_packages/refactor_provider_packages.py
@@ -432,6 +432,70 @@ class RefactorBackportPackages:
.rename("airflow.providers.amazon.common.utils.email")
)
+ def refactor_elasticsearch_package(self):
+ """
+ Fixes to "elasticsearch" providers package.
+
+ Copies some of the classes used from core Airflow to "common.utils" package of
+ the provider and renames imports to use them from there.
+
+ We copy file_task_handler.py and change import as in example diff:
+
+ .. code-block:: diff
+
+ --- ./airflow/providers/elasticsearch/log/es_task_handler.py
+ +++ ./airflow/providers/elasticsearch/log/es_task_handler.py
+ @@ -24,7 +24,7 @@
+ from airflow.configuration import conf
+ from airflow.models import TaskInstance
+ from airflow.utils import timezone
+ from airflow.utils.helpers import parse_template_string
+ -from airflow.utils.log.file_task_handler import FileTaskHandler
+ +from airflow.providers.elasticsearch.common.utils.log.file_task_handler import FileTaskHandler
+ from airflow.utils.log.json_formatter import JSONFormatter
+ from airflow.utils.log.logging_mixin import LoggingMixin
+
+ """
+
+ def elasticsearch_package_filter(node: LN, capture: Capture, filename: Filename) -> bool:
+ return filename.startswith("./airflow/providers/elasticsearch/")
+
+ os.makedirs(
+ os.path.join(get_target_providers_package_folder("elasticsearch"), "common", "utils", "log"),
+ exist_ok=True,
+ )
+ copyfile(
+ os.path.join(get_source_airflow_folder(), "airflow", "utils", "__init__.py"),
+ os.path.join(get_target_providers_package_folder("elasticsearch"), "common", "__init__.py"),
+ )
+ copyfile(
+ os.path.join(get_source_airflow_folder(), "airflow", "utils", "__init__.py"),
+ os.path.join(
+ get_target_providers_package_folder("elasticsearch"), "common", "utils", "__init__.py"
+ ),
+ )
+ copyfile(
+ os.path.join(get_source_airflow_folder(), "airflow", "utils", "log", "__init__.py"),
+ os.path.join(
+ get_target_providers_package_folder("elasticsearch"), "common", "utils", "log", "__init__.py"
+ ),
+ )
+ copyfile(
+ os.path.join(get_source_airflow_folder(), "airflow", "utils", "log", "file_task_handler.py"),
+ os.path.join(
+ get_target_providers_package_folder("elasticsearch"),
+ "common",
+ "utils",
+ "log",
+ "file_task_handler.py",
+ ),
+ )
+ (
+ self.qry.select_module("airflow.utils.log.file_task_handler")
+ .filter(callback=elasticsearch_package_filter)
+ .rename("airflow.providers.elasticsearch.common.utils.log.file_task_handler")
+ )
+
def refactor_google_package(self):
r"""
Fixes to "google" providers package.
@@ -651,6 +715,7 @@ class RefactorBackportPackages:
def do_refactor(self, in_process: bool = False) -> None: # noqa
self.rename_deprecated_modules()
self.refactor_amazon_package()
+ self.refactor_elasticsearch_package()
self.refactor_google_package()
self.refactor_odbc_package()
self.remove_tags()
diff --git a/scripts/ci/provider_packages/ci_prepare_provider_readme.sh b/scripts/ci/provider_packages/ci_prepare_provider_readme.sh
index e4d0f5e..b5f7764 100755
--- a/scripts/ci/provider_packages/ci_prepare_provider_readme.sh
+++ b/scripts/ci/provider_packages/ci_prepare_provider_readme.sh
@@ -17,7 +17,7 @@
# under the License.
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
+#
build_images::prepare_ci_build
build_images::rebuild_ci_image_if_needed