[GitHub] merlimat closed pull request #3206: Patch prometheus python client due to thread leak

[GitBox <gi...@apache.org>]

merlimat closed pull request #3206: Patch prometheus python client due to thread leak
URL: https://github.com/apache/pulsar/pull/3206
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/pulsar-functions/instance/src/main/python/prometheus_client_fix.py b/pulsar-functions/instance/src/main/python/prometheus_client_fix.py
new file mode 100644
index 0000000000..17d9ffedf0
--- /dev/null
+++ b/pulsar-functions/instance/src/main/python/prometheus_client_fix.py
@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from __future__ import unicode_literals
+
+import threading
+
+from prometheus_client import core
+from prometheus_client.exposition import MetricsHandler
+
+try:
+    from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+    from SocketServer import ThreadingMixIn
+    from urllib2 import build_opener, Request, HTTPHandler
+    from urllib import quote_plus
+    from urlparse import parse_qs, urlparse
+except ImportError:
+    # Python 3
+    from http.server import BaseHTTPRequestHandler, HTTPServer
+    from socketserver import ThreadingMixIn
+    from urllib.request import build_opener, Request, HTTPHandler
+    from urllib.parse import quote_plus, parse_qs, urlparse
+
+
+class _ThreadingSimpleServer(ThreadingMixIn, HTTPServer):
+    """Thread per request HTTP server."""
+    # Make worker threads "fire and forget". Beginning with Python 3.7 this
+    # prevents a memory leak because ``ThreadingMixIn`` starts to gather all
+    # non-daemon threads in a list in order to join on them at server close.
+    # Enabling daemon threads virtually makes ``_ThreadingSimpleServer`` the
+    # same as Python 3.7's ``ThreadingHTTPServer``.
+    daemon_threads = True
+
+def start_http_server(port, addr='', registry=core.REGISTRY):
+    """Starts an HTTP server for prometheus metrics as a daemon thread"""
+    CustomMetricsHandler = MetricsHandler.factory(registry)
+    httpd = _ThreadingSimpleServer((addr, port), CustomMetricsHandler)
+    t = threading.Thread(target=httpd.serve_forever)
+    t.daemon = True
+    t.start()
\ No newline at end of file
diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py b/pulsar-functions/instance/src/main/python/python_instance_main.py
index c6bf885e04..b7b1bfce46 100644
--- a/pulsar-functions/instance/src/main/python/python_instance_main.py
+++ b/pulsar-functions/instance/src/main/python/python_instance_main.py
@@ -39,7 +39,8 @@
 import server
 import python_instance
 import util
-import prometheus_client
+# import prometheus_client
+import prometheus_client_fix
 
 from google.protobuf import json_format
 
@@ -180,7 +181,12 @@ def main():
   pyinstance.run()
   server_instance = server.serve(args.port, pyinstance)
 
-  prometheus_client.start_http_server(args.metrics_port)
+  # Cannot use latest version of prometheus client because of thread leak
+  # prometheus_client.start_http_server(args.metrics_port)
+  # Use patched version of prometheus
+  # Contains fix from https://github.com/prometheus/client_python/pull/356
+  # This can be removed one the fix in is a official prometheus client release
+  prometheus_client_fix.start_http_server(args.metrics_port)
 
   global to_run
   while to_run:


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services