You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@liminal.apache.org by av...@apache.org on 2021/01/06 07:56:09 UTC
[incubator-liminal] 01/01: Revert "[LIMINAL-5] change prints to log"
This is an automated email from the ASF dual-hosted git repository.
aviemzur pushed a commit to branch revert-19-logging
in repository https://gitbox.apache.org/repos/asf/incubator-liminal.git
commit 43cc09f830e8794a69b38112b669742da506f2f0
Author: Aviem Zur <av...@gmail.com>
AuthorDate: Wed Jan 6 09:56:02 2021 +0200
Revert "[LIMINAL-5] change prints to log"
This reverts commit 4e323809042800ca296bc65e04baa23b1dfe6b71.
---
.../helloworld/hello_world.py | 8 +++-
liminal/build/image_builder.py | 13 +++---
liminal/build/liminal_apps_builder.py | 21 +++++-----
liminal/core/environment.py | 8 ++--
liminal/core/util/files_util.py | 5 +--
liminal/kubernetes/volume_util.py | 2 +-
liminal/logging/__init__.py | 17 --------
liminal/logging/logging_setup.py | 47 ----------------------
liminal/runners/airflow/dag/liminal_dags.py | 13 +++---
scripts/liminal | 20 ++++-----
setup.py | 4 +-
.../python/test_python_server_image_builder.py | 18 ++++-----
.../build/python/test_python_image_builder.py | 5 +--
.../airflow/build/test_liminal_apps_builder.py | 1 +
.../airflow/liminal/write_outputs/write_outputs.py | 5 +--
tests/runners/airflow/tasks/test_python.py | 3 +-
16 files changed, 59 insertions(+), 131 deletions(-)
diff --git a/examples/liminal-getting-started/helloworld/hello_world.py b/examples/liminal-getting-started/helloworld/hello_world.py
index a26752c..78fbd9c 100644
--- a/examples/liminal-getting-started/helloworld/hello_world.py
+++ b/examples/liminal-getting-started/helloworld/hello_world.py
@@ -16,8 +16,14 @@
# specific language governing permissions and limitations
# under the License.
import json
+import os
print('Hello world!\n')
+# with open('/mnt/gettingstartedvol/hello_world.json') as file:
+# print(f'hello_world.json contents = {json.loads(file.readline())}')
+#
+# os.makedirs('/mnt/vol1/', exist_ok=True)
+
with open('/mnt/gettingstartedvol/hello_world_output.json', 'w') as file:
- file.write(json.dumps({'hello': 1, 'world': 2}))
+ file.write(json.dumps({'hello': 1, 'world': 2}))
\ No newline at end of file
diff --git a/liminal/build/image_builder.py b/liminal/build/image_builder.py
index 11ff9f0..a7ca13b 100644
--- a/liminal/build/image_builder.py
+++ b/liminal/build/image_builder.py
@@ -16,7 +16,6 @@
# specific language governing permissions and limitations
# under the License.
-import logging
import os
import shutil
import subprocess
@@ -46,7 +45,7 @@ class ImageBuilder:
"""
Builds source code into an image.
"""
- logging.info(f'[ ] Building image: {self.tag}')
+ print(f'[ ] Building image: {self.tag}')
temp_dir = self.__temp_dir()
@@ -68,7 +67,7 @@ class ImageBuilder:
if self._use_buildkit():
docker_build_command = f'DOCKER_BUILDKIT=1 {docker_build_command}'
- logging.info(docker_build_command)
+ print(docker_build_command)
docker_build_out = ''
try:
@@ -79,14 +78,14 @@ class ImageBuilder:
docker_build_out = e.output
raise e
finally:
- logging.info('=' * 80)
+ print('=' * 80)
for line in str(docker_build_out)[2:-3].split('\\n'):
- logging.info(line)
- logging.info('=' * 80)
+ print(line)
+ print('=' * 80)
self.__remove_dir(temp_dir)
- logging.info(f'[X] Building image: {self.tag} (Success).')
+ print(f'[X] Building image: {self.tag} (Success).')
return docker_build_out
diff --git a/liminal/build/liminal_apps_builder.py b/liminal/build/liminal_apps_builder.py
index 19b065f..dd98477 100644
--- a/liminal/build/liminal_apps_builder.py
+++ b/liminal/build/liminal_apps_builder.py
@@ -16,7 +16,6 @@
# specific language governing permissions and limitations
# under the License.
-import logging
import os
import yaml
@@ -32,7 +31,7 @@ def build_liminal_apps(path):
config_files = files_util.find_config_files(path)
for config_file in config_files:
- logging.info(f'Building artifacts for file: {config_file}')
+ print(f'Building artifacts for file: {config_file}')
base_path = os.path.dirname(config_file)
@@ -52,8 +51,7 @@ def build_liminal_apps(path):
else:
raise ValueError(f'No such task type: {task_type}')
else:
- logging.info(
- f'No source configured for task {task_name}, skipping build..')
+ print(f'No source configured for task {task_name}, skipping build..')
if 'services' in liminal_config:
for service in liminal_config['services']:
@@ -74,7 +72,7 @@ def __build_image(base_path, builder_config, builder):
tag=builder_config['image'])
builder_instance.build()
else:
- logging.info(f"No source provided for {builder_config['name']}, skipping.")
+ print(f"No source provided for {builder_config['name']}, skipping.")
def __get_task_build_class(task_type):
@@ -85,13 +83,13 @@ def __get_service_build_class(service_type):
return service_build_types.get(service_type, None)
-logging.info(f'Loading image builder implementations..')
+print(f'Loading image builder implementations..')
# TODO: add configuration for user image builders package
image_builders_package = 'liminal.build.image'
# user_image_builders_package = 'TODO: user_image_builders_package'
-TASK_BUILD_CLASSES = class_util.find_subclasses_in_packages(
+task_build_classes = class_util.find_subclasses_in_packages(
[image_builders_package],
ImageBuilder)
@@ -101,10 +99,11 @@ def get_types_dict(task_build_classes):
return {x.split(".")[-2]: c for x, c in task_build_classes.items()}
-task_build_types = get_types_dict(TASK_BUILD_CLASSES)
+task_build_types = get_types_dict(task_build_classes)
-logging.info(f'Finished loading image builder implementations: {TASK_BUILD_CLASSES}')
-logging.info(f'Loading service image builder implementations..')
+print(f'Finished loading image builder implementations: {task_build_classes}')
+
+print(f'Loading service image builder implementations..')
# TODO: add configuration for user service image builders package
service_builders_package = 'liminal.build.service'
@@ -115,4 +114,4 @@ service_build_classes = class_util.find_subclasses_in_packages(
ServiceImageBuilderMixin)
service_build_types = get_types_dict(service_build_classes)
-logging.info(f'Finished loading service image builder implementations: {service_build_classes}')
+print(f'Finished loading service image builder implementations: {service_build_classes}')
diff --git a/liminal/core/environment.py b/liminal/core/environment.py
index acea4e4..42e14a4 100644
--- a/liminal/core/environment.py
+++ b/liminal/core/environment.py
@@ -18,7 +18,7 @@
import os
import subprocess
-import logging
+from pathlib import Path
DEFAULT_DAGS_ZIP_NAME = 'liminal.zip'
DEFAULT_LIMINAL_HOME = os.path.expanduser('~/liminal_home')
@@ -29,8 +29,8 @@ LIMINAL_VERSION_PARAM_NAME = 'LIMINAL_VERSION'
def get_liminal_home():
if not os.environ.get(LIMINAL_HOME_PARAM_NAME):
- logging.info("no environment parameter called LIMINAL_HOME detected")
- logging.info(f"registering {DEFAULT_LIMINAL_HOME} as the LIMINAL_HOME directory")
+ print("no environment parameter called LIMINAL_HOME detected")
+ print(f"registering {DEFAULT_LIMINAL_HOME} as the LIMINAL_HOME directory")
os.environ[LIMINAL_HOME_PARAM_NAME] = DEFAULT_LIMINAL_HOME
return os.environ.get(LIMINAL_HOME_PARAM_NAME, DEFAULT_LIMINAL_HOME)
@@ -58,6 +58,6 @@ def get_liminal_version():
value = pip_res[pip_res.index(' @ ') + 3:]
else:
value = pip_res
- logging.info(f'LIMINAL_VERSION not set. Setting it to currently installed version: {value}')
+ print(f'LIMINAL_VERSION not set. Setting it to currently installed version: {value}')
os.environ[LIMINAL_VERSION_PARAM_NAME] = value
return os.environ.get(LIMINAL_VERSION_PARAM_NAME, 'apache-liminal')
diff --git a/liminal/core/util/files_util.py b/liminal/core/util/files_util.py
index 48ab9b7..e611005 100644
--- a/liminal/core/util/files_util.py
+++ b/liminal/core/util/files_util.py
@@ -17,15 +17,14 @@
# under the License.
import os
-import logging
def find_config_files(path):
files = []
- logging.info(path)
+ print(path)
for r, d, f in os.walk(path):
for file in f:
if os.path.basename(file) in ['liminal.yml', 'liminal.yaml']:
- logging.info(os.path.join(r, file))
+ print(os.path.join(r, file))
files.append(os.path.join(r, file))
return files
diff --git a/liminal/kubernetes/volume_util.py b/liminal/kubernetes/volume_util.py
index eff1fd0..259faa8 100644
--- a/liminal/kubernetes/volume_util.py
+++ b/liminal/kubernetes/volume_util.py
@@ -34,7 +34,7 @@ def create_local_volumes(liminal_config, base_dir):
for volume_config in volumes_config:
if 'local' in volume_config:
- logging.info(f'Creating local kubernetes volume if needed: {volume_config}')
+ print(f'Creating local kubernetes volume if needed: {volume_config}')
path = volume_config['local']['path']
if path.startswith(".."):
path = os.path.join(base_dir, path)
diff --git a/liminal/logging/__init__.py b/liminal/logging/__init__.py
deleted file mode 100644
index 217e5db..0000000
--- a/liminal/logging/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/liminal/logging/logging_setup.py b/liminal/logging/logging_setup.py
deleted file mode 100644
index 995fa02..0000000
--- a/liminal/logging/logging_setup.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import logging
-from logging.handlers import RotatingFileHandler
-
-from liminal.core import environment
-
-LIMINAL = 'liminal'
-LOGS_DIR = 'logs'
-MAX_FILE_SIZE = 10485760 # 10 MB
-
-
-def logging_initialization():
- root_logger = logging.getLogger()
-
- log_formatter = logging.Formatter(
- '[%(asctime)s] [%(filename)s:%(lineno)d] %(levelname)s - %(message)s',
- '%m-%d %H:%M:%S'
- )
-
- file_handler = RotatingFileHandler(
- f'{environment.get_liminal_home()}/{LOGS_DIR}/{LIMINAL}.log',
- maxBytes=MAX_FILE_SIZE,
- backupCount=3
- )
- root_logger.addHandler(file_handler)
- root_logger.setLevel(logging.INFO)
-
- [h.setFormatter(log_formatter) for h in root_logger.handlers]
-
- logging.info('Logging initialization completed')
diff --git a/liminal/runners/airflow/dag/liminal_dags.py b/liminal/runners/airflow/dag/liminal_dags.py
index 03911b1..3e5e1a3 100644
--- a/liminal/runners/airflow/dag/liminal_dags.py
+++ b/liminal/runners/airflow/dag/liminal_dags.py
@@ -27,7 +27,6 @@ from liminal.core.util import files_util
from liminal.runners.airflow.model.task import Task
from liminal.runners.airflow.tasks.defaults.job_end import JobEndTask
from liminal.runners.airflow.tasks.defaults.job_start import JobStartTask
-import logging
__DEPENDS_ON_PAST = 'depends_on_past'
@@ -36,13 +35,13 @@ def register_dags(configs_path):
"""
Registers pipelines in liminal yml files found in given path (recursively) as airflow DAGs.
"""
- logging.info(f'Registering DAG from path: {configs_path}')
+ print(f'Registering DAG from path: {configs_path}')
config_files = files_util.find_config_files(configs_path)
dags = []
- logging.info(f'found {len(config_files)} in path: {configs_path}')
+ print(f'found {len(config_files)} in path: {configs_path}')
for config_file in config_files:
- logging.info(f'Registering DAG for file: {config_file}')
+ print(f'Registering DAG for file: {config_file}')
with open(config_file) as stream:
config = yaml.safe_load(stream)
@@ -85,7 +84,7 @@ def register_dags(configs_path):
job_end_task = JobEndTask(dag, config, pipeline, {}, parent, 'all_done')
job_end_task.apply_task_to_dag()
- logging.info(f'registered DAG {dag.dag_id}: {dag.tasks}')
+ print(f'registered DAG {dag.dag_id}: {dag.tasks}')
globals()[pipeline_name] = dag
dags.append(dag)
@@ -93,7 +92,7 @@ def register_dags(configs_path):
return dags
-logging.info(f'Loading task implementations..')
+print(f'Loading task implementations..')
# TODO: add configuration for user tasks package
impl_packages = 'liminal.runners.airflow.tasks'
@@ -109,7 +108,7 @@ tasks_by_liminal_name = tasks_by_liminal_name(
class_util.find_subclasses_in_packages([impl_packages], Task)
)
-logging.info(f'Finished loading task implementations: {tasks_by_liminal_name}')
+print(f'Finished loading task implementations: {tasks_by_liminal_name}')
def get_task_class(task_type):
diff --git a/scripts/liminal b/scripts/liminal
index ff4767e..3c7b132 100755
--- a/scripts/liminal
+++ b/scripts/liminal
@@ -18,7 +18,6 @@
# specific language governing permissions and limitations
# under the License.
-import logging
import os
import pathlib
import shutil
@@ -28,16 +27,13 @@ import sys
import click
import yaml
-import scripts
+import scripts as s
from liminal.build import liminal_apps_builder
from liminal.core import environment
from liminal.core.util import files_util
from liminal.kubernetes import volume_util
-from liminal.logging.logging_setup import logging_initialization
from liminal.runners.airflow import dag
-logging_initialization()
-
try:
import importlib.resources as pkg_resources
except ImportError:
@@ -68,7 +64,6 @@ def build(path):
def deploy_liminal_core_internal(clean):
- # noinspection PyTypeChecker
with pkg_resources.path(dag, 'liminal_dags.py') as p:
dags_path = p
os.makedirs(environment.get_dags_dir(), exist_ok=True)
@@ -77,13 +72,13 @@ def deploy_liminal_core_internal(clean):
# initialize the env. variable which indicates to the docke compose which
# liminal to install in airflow docker
liminal_version = environment.get_liminal_version()
- logging.info(f'Deploying liminal version: {liminal_version}')
+ print(f'Deploying liminal version: {liminal_version}')
# if liminal is installed from local file - the developer needs to put it in the /scripts folder
# in which case it will end up inside the container during build
if liminal_version.find("file://") > -1:
local_file_name = os.path.basename(liminal_version)
full_path = os.path.join('/opt/airflow/dags', local_file_name)
- logging.info(
+ print(
f'Liminal was installed locally, changing the LIMINAL_VERSION parameter to {full_path}')
os.environ[environment.LIMINAL_VERSION_PARAM_NAME] = full_path
if clean:
@@ -103,7 +98,7 @@ def docker_compose_command(command_name, args):
'-p liminal --project-directory '
f'{project_dir} {command_name} {concated_args}'
]
- logging.info(run_command[0])
+ print(run_command[0])
if 'follow' in str(args):
subprocess.call(run_command, env=os.environ, shell=True)
return '', ''
@@ -159,7 +154,7 @@ def logs(follow, tail):
docker_compose_command('logs', ['--follow'])
if tail > 0:
stdout, stderr = docker_compose_command('logs', [f'--tail={tail}'])
- logging.info(stdout)
+ print(stdout)
@cli.command("start",
@@ -167,7 +162,7 @@ def logs(follow, tail):
"Make sure docker is running on your machine")
def start():
liminal_version = environment.get_liminal_version()
- logging.info(f'starting liminal version: {liminal_version}')
+ print(f'starting liminal version: {liminal_version}')
if docker_is_running():
# initialize liminal home by default
environment.get_liminal_home()
@@ -175,8 +170,7 @@ def start():
def get_docker_compose_paths():
- # noinspection PyTypeChecker
- with pkg_resources.path(scripts, 'docker-compose.yml') as p:
+ with pkg_resources.path(s, 'docker-compose.yml') as p:
docker_compose_path = p
project_dir = pathlib.Path(docker_compose_path).parent.parent.absolute()
return docker_compose_path, project_dir
diff --git a/setup.py b/setup.py
index 6bc9845..b06fd17 100644
--- a/setup.py
+++ b/setup.py
@@ -17,7 +17,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import logging
import os
import setuptools
@@ -27,7 +26,8 @@ with open("README.md", "r") as fh:
with open('requirements.txt') as f:
requirements = f.read().splitlines()
- logging.info(requirements)
+ print(requirements)
+
setuptools.setup(
name="apache-liminal",
diff --git a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
index 88f9663..18215d3 100644
--- a/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
+++ b/tests/runners/airflow/build/http/python/test_python_server_image_builder.py
@@ -17,7 +17,6 @@
# under the License.
import json
-import logging
import os
import threading
import time
@@ -27,9 +26,8 @@ from unittest import TestCase
import docker
-from liminal.build.python import PythonImageVersions
from liminal.build.service.python_server.python_server import PythonServerImageBuilder
-
+from liminal.build.python import PythonImageVersions
class TestPythonServer(TestCase):
@@ -83,7 +81,7 @@ class TestPythonServer(TestCase):
time.sleep(5)
- logging.info('Sending request to server')
+ print('Sending request to server')
json_string = '{"key1": "val1", "key2": "val2"}'
@@ -94,33 +92,33 @@ class TestPythonServer(TestCase):
data=json_string.encode(encoding)
).read().decode(encoding))
- logging.info(f'Response from server: {server_response}')
+ print(f'Response from server: {server_response}')
self.assertEqual(f'Input was: {json.loads(json_string)}', server_response)
return build_out
def __remove_containers(self):
- logging.info(f'Stopping containers with image: {self.image_name}')
+ print(f'Stopping containers with image: {self.image_name}')
all_containers = self.docker_client.containers
matching_containers = all_containers.list(filters={'ancestor': self.image_name})
for container in matching_containers:
container_id = container.id
- logging.info(f'Stopping container {container_id}')
+ print(f'Stopping container {container_id}')
self.docker_client.api.stop(container_id)
- logging.info(f'Removing container {container_id}')
+ print(f'Removing container {container_id}')
self.docker_client.api.remove_container(container_id)
self.docker_client.containers.prune()
def __run_container(self, image_name):
try:
- logging.info(f'Running container for image: {image_name}')
+ print(f'Running container for image: {image_name}')
self.docker_client.containers.run(image_name, ports={'80/tcp': 9294})
except Exception as err:
- logging.exception(err)
+ print(err)
pass
@staticmethod
diff --git a/tests/runners/airflow/build/python/test_python_image_builder.py b/tests/runners/airflow/build/python/test_python_image_builder.py
index 7c0032a..56667bd 100644
--- a/tests/runners/airflow/build/python/test_python_image_builder.py
+++ b/tests/runners/airflow/build/python/test_python_image_builder.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import logging
import os
import shutil
import tempfile
@@ -42,7 +41,7 @@ class TestPythonImageBuilder(TestCase):
self.__remove_dir(self.temp_airflow_dir)
def test_build(self):
- for python_version in [None,
+ for python_version in [None ,
PythonImageVersions().supported_versions[0]]:
build_out = self.__test_build(python_version=python_version)
self.assertTrue('RUN pip install -r requirements.txt' in build_out,
@@ -104,7 +103,7 @@ class TestPythonImageBuilder(TestCase):
docker_client.close()
- logging.info(container_log)
+ print(container_log)
self.assertEqual(
"b'"
diff --git a/tests/runners/airflow/build/test_liminal_apps_builder.py b/tests/runners/airflow/build/test_liminal_apps_builder.py
index c011a5c..d8407e4 100644
--- a/tests/runners/airflow/build/test_liminal_apps_builder.py
+++ b/tests/runners/airflow/build/test_liminal_apps_builder.py
@@ -23,6 +23,7 @@ import docker
from liminal.build import liminal_apps_builder
+
class TestLiminalAppsBuilder(TestCase):
__image_names = [
'my_python_task_img',
diff --git a/tests/runners/airflow/liminal/write_outputs/write_outputs.py b/tests/runners/airflow/liminal/write_outputs/write_outputs.py
index 4452187..806ce0e 100644
--- a/tests/runners/airflow/liminal/write_outputs/write_outputs.py
+++ b/tests/runners/airflow/liminal/write_outputs/write_outputs.py
@@ -17,7 +17,6 @@
# under the License.
import json
-import logging
import os
split_id = int(os.environ['LIMINAL_SPLIT_ID'])
@@ -29,7 +28,7 @@ outputs_dir = '/mnt/vol1/outputs/'
if not os.path.exists(outputs_dir):
os.makedirs(outputs_dir)
-logging.info(f'Running write_outputs for split id {split_id} [NUM_SPLITS = {num_splits}]')
+print(f'Running write_outputs for split id {split_id} [NUM_SPLITS = {num_splits}]')
for filename in os.listdir(inputs_dir):
with open(os.path.join(inputs_dir, filename)) as infile, \
@@ -37,6 +36,6 @@ for filename in os.listdir(inputs_dir):
outputs_dir,
filename.replace('input', 'output').replace('.json', '.txt')
), 'w') as outfile:
- logging.info(f'Writing output file: {outfile.name}')
+ print(f'Writing output file: {outfile.name}')
data = json.loads(infile.read())
outfile.write(data['mykey'])
diff --git a/tests/runners/airflow/tasks/test_python.py b/tests/runners/airflow/tasks/test_python.py
index 1a91973..418e2eb 100644
--- a/tests/runners/airflow/tasks/test_python.py
+++ b/tests/runners/airflow/tasks/test_python.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import logging
import os
import tempfile
import unittest
@@ -59,7 +58,7 @@ class TestPythonTask(TestCase):
task1.apply_task_to_dag()
for task in dag.tasks:
- logging.info(f'Executing task {task.task_id}')
+ print(f'Executing task {task.task_id}')
task.execute({})
inputs_dir = os.path.join(self.temp_dir, 'inputs')