You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2022/04/13 16:57:29 UTC

[calcite] branch master updated: [CALCITE-4992] Resource leak in Elasticsearch adapter

This is an automated email from the ASF dual-hosted git repository.

jhyde pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git


The following commit(s) were added to refs/heads/master by this push:
     new 4cc2d80b1 [CALCITE-4992] Resource leak in Elasticsearch adapter
4cc2d80b1 is described below

commit 4cc2d80b1fa96b18285cfc494869568e7576e54c
Author: James Turton <ja...@somecomputer.xyz>
AuthorDate: Sun Mar 20 15:21:25 2022 +0200

    [CALCITE-4992] Resource leak in Elasticsearch adapter
    
    The Elasticsearch adapter was not calling 'close()' on
    RestClient instances, which is considered a resource leak.
    To solve, this change adds a Guava cache of RestClient
    objects in ElasticsearchTransport (one per endpoint), and
    the cache closes them after an hour.
    
    Close apache/calcite#2698
---
 .../elasticsearch/ElasticsearchSchemaFactory.java  | 78 +++++++++++++++++-----
 1 file changed, 63 insertions(+), 15 deletions(-)

diff --git a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchemaFactory.java b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchemaFactory.java
index 8a1f34089..afb49fa5d 100644
--- a/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchemaFactory.java
+++ b/elasticsearch/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchSchemaFactory.java
@@ -31,6 +31,11 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import com.google.common.collect.ImmutableList;
 
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
@@ -41,6 +46,8 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 
 /**
@@ -53,6 +60,37 @@ public class ElasticsearchSchemaFactory implements SchemaFactory {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchSchemaFactory.class);
 
+  private static final int REST_CLIENT_CACHE_SIZE = 100;
+
+  // RestClient objects allocate system resources and are thread safe. Here, we cache
+  // them using a key derived from the parameters that define a RestClient. The primary
+  // reason to do this is to limit the resource leak that results from Calcite's
+  // current inability to close clients that it creates.  Amongst the OS resources
+  // leaked are file descriptors which are limited to 1024 per process by default on
+  // Linux at the time of writing.
+  private static final Cache<List, RestClient> REST_CLIENTS = CacheBuilder.newBuilder()
+      .maximumSize(REST_CLIENT_CACHE_SIZE)
+      .removalListener(new RemovalListener<List, RestClient>() {
+        @Override public void onRemoval(RemovalNotification<List, RestClient> notice) {
+          LOGGER.warn(
+              "Will close an ES REST client to keep the number of open clients under {}. "
+              + "Any schema objects that might still have been relying on this client are now "
+              + "broken! Do not try to access more than {} distinct ES REST APIs through this "
+              + "adapter.",
+              REST_CLIENT_CACHE_SIZE,
+              REST_CLIENT_CACHE_SIZE
+          );
+
+          try {
+            // Free resources allocated by this RestClient
+            notice.getValue().close();
+          } catch (IOException ex) {
+            LOGGER.warn("Could not close RestClient {}", notice.getValue(), ex);
+          }
+        }
+      })
+      .build();
+
   public ElasticsearchSchemaFactory() {
   }
 
@@ -110,28 +148,38 @@ public class ElasticsearchSchemaFactory implements SchemaFactory {
    * @param hosts list of ES HTTP Hosts to connect to
    * @param username the username of ES
    * @param password the password of ES
-   * @return newly initialized low-level rest http client for ES
+   * @return new or cached low-level rest http client for ES
    */
   private static RestClient connect(List<HttpHost> hosts, String pathPrefix,
                                     String username, String password) {
 
     Objects.requireNonNull(hosts, "hosts or coordinates");
     Preconditions.checkArgument(!hosts.isEmpty(), "no ES hosts specified");
+    // Two lists are considered equal when all of their corresponding elements are equal
+    // making a list of RestClient parms a suitable cache key.
+    List cacheKey = ImmutableList.of(hosts, pathPrefix, username, password);
 
-    RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[hosts.size()]));
-
-    if (!Strings.isNullOrEmpty(username) && !Strings.isNullOrEmpty(password)) {
-      CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-      credentialsProvider.setCredentials(AuthScope.ANY,
-          new UsernamePasswordCredentials(username, password));
-      builder.setHttpClientConfigCallback(httpClientBuilder ->
-          httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
-    }
-
-    if (pathPrefix != null && !pathPrefix.isEmpty()) {
-      builder.setPathPrefix(pathPrefix);
+    try {
+      return REST_CLIENTS.get(cacheKey, new Callable<RestClient>() {
+        @Override public RestClient call() {
+          RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[hosts.size()]));
+
+          if (!Strings.isNullOrEmpty(username) && !Strings.isNullOrEmpty(password)) {
+            CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider.setCredentials(AuthScope.ANY,
+                new UsernamePasswordCredentials(username, password));
+            builder.setHttpClientConfigCallback(httpClientBuilder ->
+                httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+          }
+
+          if (pathPrefix != null && !pathPrefix.isEmpty()) {
+            builder.setPathPrefix(pathPrefix);
+          }
+          return builder.build();
+        }
+      });
+    } catch (ExecutionException ex) {
+      throw new RuntimeException("Cannot return a cached RestClient", ex);
     }
-    return builder.build();
   }
-
 }