You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2018/04/12 05:03:30 UTC
[GitHub] merlimat closed pull request #1556: fixing and standardizing
logging in functions
merlimat closed pull request #1556: fixing and standardizing logging in functions
URL: https://github.com/apache/incubator-pulsar/pull/1556
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/pulsar-functions/instance/src/main/python/log.py b/pulsar-functions/instance/src/main/python/log.py
index 18bde78270..c1a1420d09 100644
--- a/pulsar-functions/instance/src/main/python/log.py
+++ b/pulsar-functions/instance/src/main/python/log.py
@@ -22,6 +22,8 @@
''' log.py '''
import logging
+import os
+import errno
from logging.handlers import RotatingFileHandler
import pulsar
@@ -88,6 +90,13 @@ def init_rotating_logger(level, logfile, max_files, max_bytes):
It also makes sure that any StreamHandler is removed, so as to avoid stdout/stderr
constipation issues
"""
+ # create log directory if necessary
+ try:
+ os.makedirs(os.path.dirname(logfile))
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
logging.basicConfig()
root_logger = logging.getLogger()
diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py b/pulsar-functions/instance/src/main/python/python_instance_main.py
index 87ba54910b..2638fa5669 100644
--- a/pulsar-functions/instance/src/main/python/python_instance_main.py
+++ b/pulsar-functions/instance/src/main/python/python_instance_main.py
@@ -36,6 +36,7 @@
import log
import server
import python_instance
+import util
to_run = True
Log = log.Log
@@ -76,7 +77,9 @@ def main():
parser.add_argument('--log_topic', required=False, help='Topic to send Log Messages')
args = parser.parse_args()
- log_file = os.path.join(args.logging_directory, args.logging_file + ".log.0")
+ log_file = os.path.join(args.logging_directory,
+ util.getFullyQualifiedFunctionName(args.tenant, args.namespace, args.name),
+ "%s-%s.log" % (args.logging_file, args.instance_id))
log.init_rotating_logger(level=logging.INFO, logfile=log_file,
max_files=5, max_bytes=10 * 1024 * 1024)
diff --git a/pulsar-functions/instance/src/main/python/util.py b/pulsar-functions/instance/src/main/python/util.py
index 643cbacc7f..6aa4948ec4 100644
--- a/pulsar-functions/instance/src/main/python/util.py
+++ b/pulsar-functions/instance/src/main/python/util.py
@@ -64,3 +64,6 @@ def import_class_from_path(from_path, full_class_name):
Log.info("Import failed class_name %s from path %s" % (class_name, from_path))
Log.info(e, exc_info=True)
return None
+
+def getFullyQualifiedFunctionName(tenant, namespace, name):
+ return "%s/%s/%s" % (tenant, namespace, name)
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java
index 6f57ccbf09..31673d2ef7 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntimeFactory.java
@@ -79,6 +79,7 @@ public ProcessRuntimeFactory(String pulsarServiceUrl,
this.logDirectory = Paths.get("logs").toFile().getAbsolutePath();
}
}
+ this.logDirectory = this.logDirectory + "/functions";
}
@Override
diff --git a/pulsar-functions/runtime/src/main/resources/java_instance_log4j2.yml b/pulsar-functions/runtime/src/main/resources/java_instance_log4j2.yml
index 2f184d19d1..25583a61f6 100644
--- a/pulsar-functions/runtime/src/main/resources/java_instance_log4j2.yml
+++ b/pulsar-functions/runtime/src/main/resources/java_instance_log4j2.yml
@@ -116,10 +116,10 @@ Configuration:
-
RollingFile:
name: "Rolling-${ctx:function}"
- fileName : "${sys:pulsar.log.dir}/functions/${ctx:function}/function.log"
- filePattern : "${sys:pulsar.log.dir}/functions/${ctx:function}-%d{MM-dd-yyyy}-%i.log.gz"
+ fileName : "${sys:pulsar.log.dir}/${ctx:function}/${sys:pulsar.log.file}-${ctx:instance}.log"
+ filePattern : "${sys:pulsar.log.dir}/${ctx:function}/${sys:pulsar.log.file}-${ctx:instance}-%d{MM-dd-yyyy}-%i.log.gz"
PatternLayout:
- Pattern: "%d{ABSOLUTE} %level{length=5} [%thread] [instance: %X{instance}] %logger{1} - %msg%n"
+ Pattern: "%d{ABSOLUTE} %level{length=5} [%thread] [instance-%X{instance}] %logger{1} - %msg%n"
Policies:
TimeBasedTriggeringPolicy:
interval: 1
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java
index e316e6e3c0..d19d57d7b4 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/ProcessRuntimeTest.java
@@ -104,7 +104,7 @@ public void testJavaConstructor() {
assertEquals(args.size(), 41);
args.remove(args.size() - 1);
String expectedArgs = "java -cp " + javaInstanceJarFile + " -Dlog4j.configurationFile=java_instance_log4j2.yml "
- + "-Dpulsar.log.dir=" + logDirectory + " -Dpulsar.log.file=" + config.getFunctionConfig().getName()
+ + "-Dpulsar.log.dir=" + logDirectory + "/functions" + " -Dpulsar.log.file=" + config.getFunctionConfig().getName()
+ " org.apache.pulsar.functions.runtime.JavaInstanceMain"
+ " --jar " + userJarFile + " --instance_id "
+ config.getInstanceId() + " --function_id " + config.getFunctionId()
@@ -133,7 +133,7 @@ public void testPythonConstructor() {
args.remove(args.size() - 1);
String expectedArgs = "python " + pythonInstanceFile
+ " --py " + userJarFile + " --logging_directory "
- + logDirectory + " --logging_file " + config.getFunctionConfig().getName() + " --instance_id "
+ + logDirectory + "/functions" + " --logging_file " + config.getFunctionConfig().getName() + " --instance_id "
+ config.getInstanceId() + " --function_id " + config.getFunctionId()
+ " --function_version " + config.getFunctionVersion() + " --tenant " + config.getFunctionConfig().getTenant()
+ " --namespace " + config.getFunctionConfig().getNamespace()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services