You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/12/16 16:58:35 UTC

[3/6] lucene-solr:master: Merge jira/http2 branch to master

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f80e8e11/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.java
new file mode 100644
index 0000000..ac7449b
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.java
@@ -0,0 +1,690 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.Update;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.util.InputStreamResponseListener;
+import org.eclipse.jetty.http.HttpStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+/**
+ * @lucene.experimental
+ */
+public class ConcurrentUpdateHttp2SolrClient extends SolrClient {
+  private static final long serialVersionUID = 1L;
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  private static final Update END_UPDATE = new Update(null, null);
+
+  private Http2SolrClient client;
+  private final String basePath;
+  private final CustomBlockingQueue<Update> queue;
+  private final ExecutorService scheduler;
+  private final Queue<Runner> runners;
+  private final int threadCount;
+
+  private boolean shutdownClient;
+  private boolean shutdownExecutor;
+  private int pollQueueTime = 250;
+  private final boolean streamDeletes;
+  private volatile boolean closed;
+  private volatile CountDownLatch lock = null; // used to block everything
+
+  private static class CustomBlockingQueue<E> implements Iterable<E>{
+    private final BlockingQueue<E> queue;
+    private final Semaphore available;
+    private final int queueSize;
+    private final E backdoorE;
+
+    public CustomBlockingQueue(int queueSize, int maxConsumers, E backdoorE) {
+      queue = new LinkedBlockingQueue<>();
+      available = new Semaphore(queueSize);
+      this.queueSize = queueSize;
+      this.backdoorE = backdoorE;
+    }
+
+    public boolean offer(E e) {
+      boolean success = available.tryAcquire();
+      if (success) {
+        queue.offer(e);
+      }
+      return success;
+    }
+
+    public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException {
+      boolean success = available.tryAcquire(timeout, unit);
+      if (success) {
+        queue.offer(e);
+      }
+      return success;
+    }
+
+    public boolean isEmpty() {
+      return size() == 0;
+    }
+
+    public E poll(int timeout, TimeUnit unit) throws InterruptedException {
+      E e = queue.poll(timeout, unit);
+      if (e == null) {
+        return null;
+      }
+      if (e == backdoorE)
+        return null;
+      available.release();
+      return e;
+    }
+
+    public boolean add(E e) {
+      boolean success = available.tryAcquire();
+      if (success) {
+        queue.add(e);
+      } else {
+        throw new IllegalStateException("Queue is full");
+      }
+      return true;
+    }
+
+    public int size() {
+      return queueSize - available.availablePermits();
+    }
+
+    public int remainingCapacity() {
+      return available.availablePermits();
+    }
+
+    @Override
+    public Iterator<E> iterator() {
+      return queue.iterator();
+    }
+
+    public void backdoorOffer() {
+      queue.offer(backdoorE);
+    }
+  }
+
+  protected ConcurrentUpdateHttp2SolrClient(Builder builder) {
+    this.client = builder.client;
+    this.shutdownClient = builder.closeHttp2Client;
+    this.threadCount = builder.threadCount;
+    this.queue = new CustomBlockingQueue<>(builder.queueSize, threadCount, END_UPDATE);
+    this.runners = new LinkedList<>();
+    this.streamDeletes = builder.streamDeletes;
+    this.basePath = builder.baseSolrUrl;
+
+    if (builder.executorService != null) {
+      this.scheduler = builder.executorService;
+      this.shutdownExecutor = false;
+    } else {
+      this.scheduler = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrjNamedThreadFactory("concurrentUpdateScheduler"));
+      this.shutdownExecutor = true;
+    }
+
+  }
+
+  /**
+   * Opens a connection and sends everything...
+   */
+  class Runner implements Runnable {
+
+    @Override
+    public void run() {
+      log.debug("starting runner: {}", this);
+      // This loop is so we can continue if an element was added to the queue after the last runner exited.
+      for (;;) {
+        try {
+
+          sendUpdateStream();
+
+        } catch (Throwable e) {
+          if (e instanceof OutOfMemoryError) {
+            throw (OutOfMemoryError) e;
+          }
+          handleError(e);
+        } finally {
+          synchronized (runners) {
+            // check to see if anything else was added to the queue
+            if (runners.size() == 1 && !queue.isEmpty() && !scheduler.isShutdown()) {
+              // If there is something else to process, keep last runner alive by staying in the loop.
+            } else {
+              runners.remove(this);
+              if (runners.isEmpty()) {
+                // notify anyone waiting in blockUntilFinished
+                runners.notifyAll();
+              }
+              break;
+            }
+          }
+        }
+      }
+
+      log.debug("finished: {}", this);
+    }
+
+    //
+    // Pull from the queue multiple times and streams over a single connection.
+    // Exits on exception, interruption, or an empty queue to pull from.
+    //
+    void sendUpdateStream() throws Exception {
+
+      try {
+        while (!queue.isEmpty()) {
+          InputStream rspBody = null;
+          try {
+            Update update;
+            notifyQueueAndRunnersIfEmptyQueue();
+            update = queue.poll(pollQueueTime, TimeUnit.MILLISECONDS);
+
+            if (update == null) {
+              break;
+            }
+
+            InputStreamResponseListener responseListener = null;
+            try (Http2SolrClient.OutStream out = client.initOutStream(basePath, update.getRequest(),
+                update.getCollection())) {
+              Update upd = update;
+              while (upd != null) {
+                UpdateRequest req = upd.getRequest();
+                if (!out.belongToThisStream(req, upd.getCollection())) {
+                  queue.add(upd); // Request has different params or destination core/collection, return to queue
+                  break;
+                }
+                client.send(out, upd.getRequest(), upd.getCollection());
+                out.flush();
+
+                notifyQueueAndRunnersIfEmptyQueue();
+                upd = queue.poll(pollQueueTime, TimeUnit.MILLISECONDS);
+              }
+              responseListener = out.getResponseListener();
+            }
+
+            Response response = responseListener.get(client.getIdleTimeout(), TimeUnit.MILLISECONDS);
+            rspBody = responseListener.getInputStream();
+
+            int statusCode = response.getStatus();
+            if (statusCode != HttpStatus.OK_200) {
+              StringBuilder msg = new StringBuilder();
+              msg.append(response.getReason());
+              msg.append("\n\n\n\n");
+              msg.append("request: ").append(basePath);
+
+              SolrException solrExc;
+              NamedList<String> metadata = null;
+              // parse out the metadata from the SolrException
+              try {
+                String encoding = "UTF-8"; // default
+                NamedList<Object> resp = client.getParser().processResponse(rspBody, encoding);
+                NamedList<Object> error = (NamedList<Object>) resp.get("error");
+                if (error != null) {
+                  metadata = (NamedList<String>) error.get("metadata");
+                  String remoteMsg = (String) error.get("msg");
+                  if (remoteMsg != null) {
+                    msg.append("\nRemote error message: ");
+                    msg.append(remoteMsg);
+                  }
+                }
+              } catch (Exception exc) {
+                // don't want to fail to report error if parsing the response fails
+                log.warn("Failed to parse error response from " + basePath + " due to: " + exc);
+              } finally {
+                solrExc = new HttpSolrClient.RemoteSolrException(basePath , statusCode, msg.toString(), null);
+                if (metadata != null) {
+                  solrExc.setMetadata(metadata);
+                }
+              }
+
+              handleError(solrExc);
+            } else {
+              onSuccess(response, rspBody);
+            }
+
+          } finally {
+            try {
+              if (rspBody != null) {
+                while (rspBody.read() != -1) {}
+              }
+            } catch (Exception e) {
+              log.error("Error consuming and closing http response stream.", e);
+            }
+            notifyQueueAndRunnersIfEmptyQueue();
+          }
+        }
+      } catch (InterruptedException e) {
+        log.error("Interrupted on polling from queue", e);
+      }
+
+    }
+  }
+
+  private void notifyQueueAndRunnersIfEmptyQueue() {
+    if (queue.size() == 0) {
+      synchronized (queue) {
+        // queue may be empty
+        queue.notifyAll();
+      }
+      synchronized (runners) {
+        // we notify runners too - if there is a high queue poll time and this is the update
+        // that emptied the queue, we make an attempt to avoid the 250ms timeout in blockUntilFinished
+        runners.notifyAll();
+      }
+    }
+  }
+
+  // *must* be called with runners monitor held, e.g. synchronized(runners){ addRunner() }
+  private void addRunner() {
+    MDC.put("ConcurrentUpdateHttp2SolrClient.url", client.getBaseURL());
+    try {
+      Runner r = new Runner();
+      runners.add(r);
+      try {
+        scheduler.execute(r);  // this can throw an exception if the scheduler has been shutdown, but that should be fine.
+      } catch (RuntimeException e) {
+        runners.remove(r);
+        throw e;
+      }
+    } finally {
+      MDC.remove("ConcurrentUpdateHttp2SolrClient.url");
+    }
+  }
+
+  @Override
+  public NamedList<Object> request(final SolrRequest request, String collection)
+      throws SolrServerException, IOException {
+    if (!(request instanceof UpdateRequest)) {
+      request.setBasePath(basePath);
+      return client.request(request, collection);
+    }
+    UpdateRequest req = (UpdateRequest) request;
+    req.setBasePath(basePath);
+    // this happens for commit...
+    if (streamDeletes) {
+      if ((req.getDocuments() == null || req.getDocuments().isEmpty())
+          && (req.getDeleteById() == null || req.getDeleteById().isEmpty())
+          && (req.getDeleteByIdMap() == null || req.getDeleteByIdMap().isEmpty())) {
+        if (req.getDeleteQuery() == null) {
+          blockUntilFinished();
+          return client.request(request, collection);
+        }
+      }
+    } else {
+      if ((req.getDocuments() == null || req.getDocuments().isEmpty())) {
+        blockUntilFinished();
+        return client.request(request, collection);
+      }
+    }
+
+
+    SolrParams params = req.getParams();
+    if (params != null) {
+      // check if it is waiting for the searcher
+      if (params.getBool(UpdateParams.WAIT_SEARCHER, false)) {
+        log.info("blocking for commit/optimize");
+        blockUntilFinished(); // empty the queue
+        return client.request(request, collection);
+      }
+    }
+
+    try {
+      CountDownLatch tmpLock = lock;
+      if (tmpLock != null) {
+        tmpLock.await();
+      }
+
+      Update update = new Update(req, collection);
+      boolean success = queue.offer(update);
+
+      for (;;) {
+        synchronized (runners) {
+          // see if queue is half full and we can add more runners
+          // special case: if only using a threadCount of 1 and the queue
+          // is filling up, allow 1 add'l runner to help process the queue
+          if (runners.isEmpty() || (queue.remainingCapacity() < queue.size() && runners.size() < threadCount))
+          {
+            // We need more runners, so start a new one.
+            addRunner();
+          } else {
+            // break out of the retry loop if we added the element to the queue
+            // successfully, *and*
+            // while we are still holding the runners lock to prevent race
+            // conditions.
+            if (success)
+              break;
+          }
+        }
+
+        // Retry to add to the queue w/o the runners lock held (else we risk
+        // temporary deadlock)
+        // This retry could also fail because
+        // 1) existing runners were not able to take off any new elements in the
+        // queue
+        // 2) the queue was filled back up since our last try
+        // If we succeed, the queue may have been completely emptied, and all
+        // runners stopped.
+        // In all cases, we should loop back to the top to see if we need to
+        // start more runners.
+        //
+        if (!success) {
+          success = queue.offer(update, 100, TimeUnit.MILLISECONDS);
+        }
+      }
+    } catch (InterruptedException e) {
+      log.error("interrupted", e);
+      throw new IOException(e.getLocalizedMessage());
+    }
+
+    // RETURN A DUMMY result
+    NamedList<Object> dummy = new NamedList<>();
+    dummy.add("NOTE", "the request is processed in a background stream");
+    return dummy;
+  }
+
+  public synchronized void blockUntilFinished() {
+    lock = new CountDownLatch(1);
+    try {
+
+      waitForEmptyQueue();
+      interruptRunnerThreadsPolling();
+
+      synchronized (runners) {
+
+        // NOTE: if the executor is shut down, runners may never become empty (a scheduled task may never be run,
+        // which means it would never remove itself from the runners list. This is why we don't wait forever
+        // and periodically check if the scheduler is shutting down.
+        int loopCount = 0;
+        while (!runners.isEmpty()) {
+
+          if (scheduler.isShutdown())
+            break;
+
+          loopCount++;
+
+          // Need to check if the queue is empty before really considering this is finished (SOLR-4260)
+          int queueSize = queue.size();
+          if (queueSize > 0 && runners.isEmpty()) {
+            // TODO: can this still happen?
+            log.warn("No more runners, but queue still has " +
+                queueSize + " adding more runners to process remaining requests on queue");
+            addRunner();
+          }
+
+          interruptRunnerThreadsPolling();
+
+          // try to avoid the worst case wait timeout
+          // without bad spin
+          int timeout;
+          if (loopCount < 3) {
+            timeout = 10;
+          } else if (loopCount < 10) {
+            timeout = 25;
+          } else {
+            timeout = 250;
+          }
+
+          try {
+            runners.wait(timeout);
+          } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+          }
+        }
+      }
+    } finally {
+      lock.countDown();
+      lock = null;
+    }
+  }
+
+  private void waitForEmptyQueue() {
+    boolean threadInterrupted = Thread.currentThread().isInterrupted();
+
+    while (!queue.isEmpty()) {
+      if (scheduler.isTerminated()) {
+        log.warn("The task queue still has elements but the update scheduler {} is terminated. Can't process any more tasks. "
+            + "Queue size: {}, Runners: {}. Current thread Interrupted? {}", scheduler, queue.size(), runners.size(), threadInterrupted);
+        break;
+      }
+
+      synchronized (runners) {
+        int queueSize = queue.size();
+        if (queueSize > 0 && runners.isEmpty()) {
+          log.warn("No more runners, but queue still has " +
+              queueSize + " adding more runners to process remaining requests on queue");
+          addRunner();
+        }
+      }
+      synchronized (queue) {
+        try {
+          queue.wait(250);
+        } catch (InterruptedException e) {
+          // If we set the thread as interrupted again, the next time the wait it's called i t's going to return immediately
+          threadInterrupted = true;
+          log.warn("Thread interrupted while waiting for update queue to be empty. There are still {} elements in the queue.",
+              queue.size());
+        }
+      }
+    }
+    if (threadInterrupted) {
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  public void handleError(Throwable ex) {
+    log.error("error", ex);
+  }
+
+  /**
+   * Intended to be used as an extension point for doing post processing after a request completes.
+   */
+  public void onSuccess(Response resp, InputStream respBody) {
+    // no-op by design, override to add functionality
+  }
+
+  @Override
+  public synchronized void close() {
+    if (closed) {
+      interruptRunnerThreadsPolling();
+      return;
+    }
+    closed = true;
+
+    try {
+      if (shutdownExecutor) {
+        scheduler.shutdown();
+        interruptRunnerThreadsPolling();
+        try {
+          if (!scheduler.awaitTermination(60, TimeUnit.SECONDS)) {
+            scheduler.shutdownNow();
+            if (!scheduler.awaitTermination(60, TimeUnit.SECONDS)) log
+                .error("ExecutorService did not terminate");
+          }
+        } catch (InterruptedException ie) {
+          scheduler.shutdownNow();
+          Thread.currentThread().interrupt();
+        }
+      } else {
+        interruptRunnerThreadsPolling();
+      }
+    } finally {
+      if (shutdownClient)
+        client.close();
+    }
+  }
+
+  private void interruptRunnerThreadsPolling() {
+    synchronized (runners) {
+      for (Runner ignored : runners) {
+        queue.backdoorOffer();
+      }
+    }
+  }
+
+  public void shutdownNow() {
+    if (closed) {
+      return;
+    }
+    closed = true;
+
+    if (shutdownExecutor) {
+      scheduler.shutdown();
+      interruptRunnerThreadsPolling();
+      scheduler.shutdownNow(); // Cancel currently executing tasks
+      try {
+        if (!scheduler.awaitTermination(30, TimeUnit.SECONDS))
+          log.error("ExecutorService did not terminate");
+      } catch (InterruptedException ie) {
+        scheduler.shutdownNow();
+        Thread.currentThread().interrupt();
+      }
+    } else {
+      interruptRunnerThreadsPolling();
+    }
+  }
+
+  /**
+   * @param pollQueueTime time for an open connection to wait for updates when
+   * the queue is empty.
+   */
+  public void setPollQueueTime(int pollQueueTime) {
+    this.pollQueueTime = pollQueueTime;
+  }
+
+  /**
+   * Constructs {@link ConcurrentUpdateHttp2SolrClient} instances from provided configuration.
+   */
+  public static class Builder {
+    protected Http2SolrClient client;
+    protected String baseSolrUrl;
+    protected int queueSize = 10;
+    protected int threadCount;
+    protected ExecutorService executorService;
+    protected boolean streamDeletes;
+    protected boolean closeHttp2Client;
+
+    public Builder(String baseSolrUrl, Http2SolrClient client) {
+      this(baseSolrUrl, client, false);
+    }
+
+    public Builder(String baseSolrUrl, Http2SolrClient client, boolean closeHttp2Client) {
+      this.baseSolrUrl = baseSolrUrl;
+      this.client = client;
+      this.closeHttp2Client = closeHttp2Client;
+    }
+
+    /**
+     * The maximum number of requests buffered by the SolrClient's internal queue before being processed by background threads.
+     *
+     * This value should be carefully paired with the number of queue-consumer threads.  A queue with a maximum size
+     * set too high may require more memory.  A queue with a maximum size set too low may suffer decreased throughput
+     * as {@link ConcurrentUpdateHttp2SolrClient#request(SolrRequest)} calls block waiting to add requests to the queue.
+     *
+     * If not set, this defaults to 10.
+     *
+     * @see #withThreadCount(int)
+     */
+    public Builder withQueueSize(int queueSize) {
+      if (queueSize <= 0) {
+        throw new IllegalArgumentException("queueSize must be a positive integer.");
+      }
+      this.queueSize = queueSize;
+      return this;
+    }
+
+    /**
+     * The maximum number of threads used to empty {@link ConcurrentUpdateHttp2SolrClient}s queue.
+     *
+     * Threads are created when documents are added to the client's internal queue and exit when no updates remain in
+     * the queue.
+     * <p>
+     * This value should be carefully paired with the maximum queue capacity.  A client with too few threads may suffer
+     * decreased throughput as the queue fills up and {@link ConcurrentUpdateHttp2SolrClient#request(SolrRequest)} calls
+     * block waiting to add requests to the queue.
+     */
+    public Builder withThreadCount(int threadCount) {
+      if (threadCount <= 0) {
+        throw new IllegalArgumentException("threadCount must be a positive integer.");
+      }
+
+      this.threadCount = threadCount;
+      return this;
+    }
+
+    /**
+     * Provides the {@link ExecutorService} for the created client to use when servicing the update-request queue.
+     */
+    public Builder withExecutorService(ExecutorService executorService) {
+      this.executorService = executorService;
+      return this;
+    }
+
+    /**
+     * Configures created clients to always stream delete requests.
+     *
+     * Streamed deletes are put into the update-queue and executed like any other update request.
+     */
+    public Builder alwaysStreamDeletes() {
+      this.streamDeletes = true;
+      return this;
+    }
+
+    /**
+     * Configures created clients to not stream delete requests.
+     *
+     * With this option set when the created ConcurrentUpdateHttp2SolrClient sents a delete request it will first will lock
+     * the queue and block until all queued updates have been sent, and then send the delete request.
+     */
+    public Builder neverStreamDeletes() {
+      this.streamDeletes = false;
+      return this;
+    }
+
+    /**
+     * Create a {@link ConcurrentUpdateHttp2SolrClient} based on the provided configuration options.
+     */
+    public ConcurrentUpdateHttp2SolrClient build() {
+      if (baseSolrUrl == null) {
+        throw new IllegalArgumentException("Cannot create HttpSolrClient without a valid baseSolrUrl!");
+      }
+
+      return new ConcurrentUpdateHttp2SolrClient(this);
+    }
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f80e8e11/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
new file mode 100644
index 0000000..a50f20a
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -0,0 +1,979 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.lang.invoke.MethodHandles;
+import java.net.ConnectException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpStatus;
+import org.apache.http.entity.ContentType;
+import org.apache.solr.client.solrj.ResponseParser;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.V2RequestSupport;
+import org.apache.solr.client.solrj.embedded.SSLConfig;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.client.solrj.util.Constants;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.StringUtils;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.Base64;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.ObjectReleaseTracker;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.HttpClientTransport;
+import org.eclipse.jetty.client.ProtocolHandlers;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.api.Result;
+import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
+import org.eclipse.jetty.client.util.BufferingResponseListener;
+import org.eclipse.jetty.client.util.BytesContentProvider;
+import org.eclipse.jetty.client.util.FormContentProvider;
+import org.eclipse.jetty.client.util.InputStreamContentProvider;
+import org.eclipse.jetty.client.util.InputStreamResponseListener;
+import org.eclipse.jetty.client.util.MultiPartContentProvider;
+import org.eclipse.jetty.client.util.OutputStreamContentProvider;
+import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.http.HttpField;
+import org.eclipse.jetty.http.HttpFields;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpMethod;
+import org.eclipse.jetty.http2.client.HTTP2Client;
+import org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2;
+import org.eclipse.jetty.util.BlockingArrayQueue;
+import org.eclipse.jetty.util.Fields;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.util.Utils.getObjectByPath;
+
+// TODO: error handling, small Http2SolrClient features, security, ssl
+/**
+ * @lucene.experimental
+ */
+public class Http2SolrClient extends SolrClient {
+  public static final String REQ_PRINCIPAL_KEY = "solr-req-principal";
+
+  private static volatile SSLConfig defaultSSLConfig;
+
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  private static final String AGENT = "Solr[" + Http2SolrClient.class.getName() + "] 2.0";
+  private static final String UTF_8 = StandardCharsets.UTF_8.name();
+  private static final String DEFAULT_PATH = "/select";
+  private static final List<String> errPath = Arrays.asList("metadata", "error-class");
+
+  private HttpClient httpClient;
+  private volatile Set<String> queryParams = Collections.emptySet();
+  private int idleTimeout;
+
+  private ResponseParser parser = new BinaryResponseParser();
+  private volatile RequestWriter requestWriter = new BinaryRequestWriter();
+  private List<HttpListenerFactory> listenerFactory = new LinkedList<>();
+  private AsyncTracker asyncTracker = new AsyncTracker();
+  /**
+   * The URL of the Solr server.
+   */
+  private String serverBaseUrl;
+  private boolean closeClient;
+
+  protected Http2SolrClient(String serverBaseUrl, Builder builder) {
+    if (serverBaseUrl != null)  {
+      if (!serverBaseUrl.equals("/") && serverBaseUrl.endsWith("/")) {
+        serverBaseUrl = serverBaseUrl.substring(0, serverBaseUrl.length() - 1);
+      }
+
+      if (serverBaseUrl.startsWith("//")) {
+        serverBaseUrl = serverBaseUrl.substring(1, serverBaseUrl.length());
+      }
+      this.serverBaseUrl = serverBaseUrl;
+    }
+
+    if (builder.idleTimeout != null) idleTimeout = builder.idleTimeout;
+    else idleTimeout = HttpClientUtil.DEFAULT_SO_TIMEOUT;
+
+    if (builder.httpClient == null) {
+      httpClient = createHttpClient(builder);
+      closeClient = true;
+    } else {
+      httpClient = builder.httpClient;
+    }
+    if (!httpClient.isStarted()) {
+      try {
+        httpClient.start();
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    assert ObjectReleaseTracker.track(this);
+  }
+
+  public void addListenerFactory(HttpListenerFactory factory) {
+    this.listenerFactory.add(factory);
+  }
+
+  HttpClient getHttpClient() {
+    return httpClient;
+  }
+
+  ProtocolHandlers getProtocolHandlers() {
+    return httpClient.getProtocolHandlers();
+  }
+
+  private HttpClient createHttpClient(Builder builder) {
+    HttpClient httpClient;
+
+    BlockingArrayQueue<Runnable> queue = new BlockingArrayQueue<>(256, 256);
+    ThreadPoolExecutor httpClientExecutor = new ExecutorUtil.MDCAwareThreadPoolExecutor(32,
+        256, 60, TimeUnit.SECONDS, queue, new SolrjNamedThreadFactory("h2sc"));
+
+    SslContextFactory sslContextFactory;
+    boolean ssl;
+    if (builder.sslConfig == null) {
+      sslContextFactory = getDefaultSslContextFactory();
+      ssl = sslContextFactory.getTrustStore() != null || sslContextFactory.getTrustStorePath() != null;
+    } else {
+      sslContextFactory = builder.sslConfig.createContextFactory();
+      ssl = true;
+    }
+
+    boolean sslOnJava8OrLower = ssl && !Constants.JRE_IS_MINIMUM_JAVA9;
+    HttpClientTransport transport;
+    if (builder.useHttp1_1 || sslOnJava8OrLower) {
+      if (sslOnJava8OrLower && !builder.useHttp1_1) {
+        log.warn("Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2");
+      } else {
+        log.debug("Create Http2SolrClient with HTTP/1.1 transport");
+      }
+      transport = new HttpClientTransportOverHTTP(2);
+      httpClient = new HttpClient(transport, sslContextFactory);
+      if (builder.maxConnectionsPerHost != null) httpClient.setMaxConnectionsPerDestination(builder.maxConnectionsPerHost);
+    } else {
+      log.debug("Create Http2SolrClient with HTTP/2 transport");
+      HTTP2Client http2client = new HTTP2Client();
+      transport = new HttpClientTransportOverHTTP2(http2client);
+      httpClient = new HttpClient(transport, sslContextFactory);
+      httpClient.setMaxConnectionsPerDestination(4);
+    }
+
+    httpClient.setExecutor(httpClientExecutor);
+    httpClient.setStrictEventOrdering(false);
+    httpClient.setConnectBlocking(true);
+    httpClient.setFollowRedirects(false);
+    httpClient.setMaxRequestsQueuedPerDestination(asyncTracker.getMaxRequestsQueuedPerDestination());
+    httpClient.setUserAgentField(new HttpField(HttpHeader.USER_AGENT, AGENT));
+
+    if (builder.idleTimeout != null) httpClient.setIdleTimeout(builder.idleTimeout);
+    if (builder.connectionTimeout != null) httpClient.setConnectTimeout(builder.connectionTimeout);
+    return httpClient;
+  }
+
+  public void close() {
+    // we wait for async requests, so far devs don't want to give sugar for this
+    asyncTracker.waitForComplete();
+    if (closeClient) {
+      try {
+        ExecutorService executor = (ExecutorService) httpClient.getExecutor();
+        httpClient.setStopTimeout(1000);
+        httpClient.stop();
+        ExecutorUtil.shutdownAndAwaitTermination(executor);
+      } catch (Exception e) {
+        throw new RuntimeException("Exception on closing client", e);
+      }
+    }
+
+    assert ObjectReleaseTracker.release(this);
+    System.out.println("Done close " + httpClient.getExecutor().toString());
+  }
+
+  public boolean isV2ApiRequest(final SolrRequest request) {
+    return request instanceof V2Request || request.getPath().contains("/____v2");
+  }
+
+  public long getIdleTimeout() {
+    return idleTimeout;
+  }
+
+  public static class OutStream implements Closeable{
+    private final String origCollection;
+    private final ModifiableSolrParams origParams;
+    private final OutputStreamContentProvider outProvider;
+    private final InputStreamResponseListener responseListener;
+    private final boolean isXml;
+
+    public OutStream(String origCollection, ModifiableSolrParams origParams,
+                     OutputStreamContentProvider outProvider, InputStreamResponseListener responseListener, boolean isXml) {
+      this.origCollection = origCollection;
+      this.origParams = origParams;
+      this.outProvider = outProvider;
+      this.responseListener = responseListener;
+      this.isXml = isXml;
+    }
+
+    boolean belongToThisStream(SolrRequest solrRequest, String collection) {
+      ModifiableSolrParams solrParams = new ModifiableSolrParams(solrRequest.getParams());
+      if (!origParams.toNamedList().equals(solrParams.toNamedList()) || !StringUtils.equals(origCollection, collection)) {
+        return false;
+      }
+      return true;
+    }
+
+    public void write(byte b[]) throws IOException {
+      this.outProvider.getOutputStream().write(b);
+    }
+
+    public void flush() throws IOException {
+      this.outProvider.getOutputStream().flush();
+    }
+
+    @Override
+    public void close() throws IOException {
+      if (isXml) {
+        write("</stream>".getBytes(StandardCharsets.UTF_8));
+      }
+      this.outProvider.getOutputStream().close();
+    }
+
+    //TODO this class should be hidden
+    public InputStreamResponseListener getResponseListener() {
+      return responseListener;
+    }
+  }
+
+  public OutStream initOutStream(String baseUrl,
+                                 UpdateRequest updateRequest,
+                                 String collection) throws IOException {
+    String contentType = requestWriter.getUpdateContentType();
+    final ModifiableSolrParams origParams = new ModifiableSolrParams(updateRequest.getParams());
+
+    // The parser 'wt=' and 'version=' params are used instead of the
+    // original params
+    ModifiableSolrParams requestParams = new ModifiableSolrParams(origParams);
+    requestParams.set(CommonParams.WT, parser.getWriterType());
+    requestParams.set(CommonParams.VERSION, parser.getVersion());
+
+    String basePath = baseUrl;
+    if (collection != null)
+      basePath += "/" + collection;
+    if (!basePath.endsWith("/"))
+      basePath += "/";
+
+    OutputStreamContentProvider provider = new OutputStreamContentProvider();
+    Request postRequest = httpClient
+        .newRequest(basePath + "update"
+            + requestParams.toQueryString())
+        .method(HttpMethod.POST)
+        .header("User-Agent", HttpSolrClient.AGENT)
+        .header("Content-Type", contentType)
+        .content(provider);
+    setListeners(updateRequest, postRequest);
+    InputStreamResponseListener responseListener = new InputStreamResponseListener();
+    postRequest.send(responseListener);
+
+    boolean isXml = ClientUtils.TEXT_XML.equals(requestWriter.getUpdateContentType());
+    OutStream outStream = new OutStream(collection, origParams, provider, responseListener,
+        isXml);
+    if (isXml) {
+      outStream.write("<stream>".getBytes(StandardCharsets.UTF_8));
+    }
+    return outStream;
+  }
+
+  public void send(OutStream outStream, SolrRequest req, String collection) throws IOException {
+    assert outStream.belongToThisStream(req, collection);
+    this.requestWriter.write(req, outStream.outProvider.getOutputStream());
+    if (outStream.isXml) {
+      // check for commit or optimize
+      SolrParams params = req.getParams();
+      if (params != null) {
+        String fmt = null;
+        if (params.getBool(UpdateParams.OPTIMIZE, false)) {
+          fmt = "<optimize waitSearcher=\"%s\" />";
+        } else if (params.getBool(UpdateParams.COMMIT, false)) {
+          fmt = "<commit waitSearcher=\"%s\" />";
+        }
+        if (fmt != null) {
+          byte[] content = String.format(Locale.ROOT,
+              fmt, params.getBool(UpdateParams.WAIT_SEARCHER, false)
+                  + "")
+              .getBytes(StandardCharsets.UTF_8);
+          outStream.write(content);
+        }
+      }
+    }
+    outStream.flush();
+  }
+
+  public NamedList<Object> request(SolrRequest solrRequest,
+                                      String collection,
+                                      OnComplete onComplete) throws IOException, SolrServerException {
+    Request req = makeRequest(solrRequest, collection);
+    final ResponseParser parser = solrRequest.getResponseParser() == null
+        ? this.parser: solrRequest.getResponseParser();
+
+    if (onComplete != null) {
+      // This async call only suitable for indexing since the response size is limited by 5MB
+      req.onRequestQueued(asyncTracker.queuedListener)
+          .onComplete(asyncTracker.completeListener).send(new BufferingResponseListener(5 * 1024 * 1024) {
+
+        @Override
+        public void onComplete(Result result) {
+          if (result.isFailed()) {
+            onComplete.onFailure(result.getFailure());
+            return;
+          }
+
+          NamedList<Object> rsp;
+          try {
+            InputStream is = getContentAsInputStream();
+            assert ObjectReleaseTracker.track(is);
+            rsp = processErrorsAndResponse(result.getResponse(),
+                parser, is, getEncoding(), isV2ApiRequest(solrRequest));
+            onComplete.onSuccess(rsp);
+          } catch (Exception e) {
+            onComplete.onFailure(e);
+          }
+        }
+      });
+      return null;
+    } else {
+      try {
+        InputStreamResponseListener listener = new InputStreamResponseListener();
+        req.send(listener);
+        Response response = listener.get(idleTimeout, TimeUnit.MILLISECONDS);
+        InputStream is = listener.getInputStream();
+        assert ObjectReleaseTracker.track(is);
+        return processErrorsAndResponse(response, parser, is, getEncoding(response), isV2ApiRequest(solrRequest));
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        throw new RuntimeException(e);
+      } catch (TimeoutException e) {
+        throw new SolrServerException(
+            "Timeout occured while waiting response from server at: " + req.getURI(), e);
+      } catch (ExecutionException e) {
+        Throwable cause = e.getCause();
+        if (cause instanceof ConnectException) {
+          throw new SolrServerException("Server refused connection at: " + req.getURI(), cause);
+        }
+        if (cause instanceof SolrServerException) {
+          throw (SolrServerException) cause;
+        } else if (cause instanceof IOException) {
+          throw new SolrServerException(
+              "IOException occured when talking to server at: " + getBaseURL(), cause);
+        }
+        throw new SolrServerException(cause.getMessage(), cause);
+      }
+    }
+  }
+
+  private String getEncoding(Response response) {
+    String contentType = response.getHeaders().get(HttpHeader.CONTENT_TYPE);
+    if (contentType != null) {
+      String charset = "charset=";
+      int index = contentType.toLowerCase(Locale.ENGLISH).indexOf(charset);
+      if (index > 0) {
+        String encoding = contentType.substring(index + charset.length());
+        // Sometimes charsets arrive with an ending semicolon.
+        int semicolon = encoding.indexOf(';');
+        if (semicolon > 0)
+          encoding = encoding.substring(0, semicolon).trim();
+        // Sometimes charsets are quoted.
+        int lastIndex = encoding.length() - 1;
+        if (encoding.charAt(0) == '"' && encoding.charAt(lastIndex) == '"')
+          encoding = encoding.substring(1, lastIndex).trim();
+        return encoding;
+      }
+    }
+    return null;
+  }
+
+  private void setBasicAuthHeader(SolrRequest solrRequest, Request req) {
+    if (solrRequest.getBasicAuthUser() != null && solrRequest.getBasicAuthPassword() != null) {
+      String userPass = solrRequest.getBasicAuthUser() + ":" + solrRequest.getBasicAuthPassword();
+      String encoded = Base64.byteArrayToBase64(userPass.getBytes(StandardCharsets.UTF_8));
+      req.header("Authorization", "Basic " + encoded);
+    }
+  }
+
+  private Request makeRequest(SolrRequest solrRequest, String collection)
+      throws SolrServerException, IOException {
+    Request req = createRequest(solrRequest, collection);
+    setListeners(solrRequest, req);
+    if (solrRequest.getUserPrincipal() != null) {
+      req.attribute(REQ_PRINCIPAL_KEY, solrRequest.getUserPrincipal());
+    }
+
+    return req;
+  }
+
+  private void setListeners(SolrRequest solrRequest, Request req) {
+    setBasicAuthHeader(solrRequest, req);
+    for (HttpListenerFactory factory : listenerFactory) {
+      HttpListenerFactory.RequestResponseListener listener = factory.get();
+      req.onRequestQueued(listener);
+      req.onRequestBegin(listener);
+      req.onComplete(listener);
+    }
+  }
+
+  private Request createRequest(SolrRequest solrRequest, String collection) throws IOException, SolrServerException {
+    if (solrRequest.getBasePath() == null && serverBaseUrl == null)
+      throw new IllegalArgumentException("Destination node is not provided!");
+
+    if (solrRequest instanceof V2RequestSupport) {
+      solrRequest = ((V2RequestSupport) solrRequest).getV2Request();
+    }
+    SolrParams params = solrRequest.getParams();
+    RequestWriter.ContentWriter contentWriter = requestWriter.getContentWriter(solrRequest);
+    Collection<ContentStream> streams = contentWriter == null ? requestWriter.getContentStreams(solrRequest) : null;
+    String path = requestWriter.getPath(solrRequest);
+    if (path == null || !path.startsWith("/")) {
+      path = DEFAULT_PATH;
+    }
+
+    ResponseParser parser = solrRequest.getResponseParser();
+    if (parser == null) {
+      parser = this.parser;
+    }
+
+    // The parser 'wt=' and 'version=' params are used instead of the original
+    // params
+    ModifiableSolrParams wparams = new ModifiableSolrParams(params);
+    if (parser != null) {
+      wparams.set(CommonParams.WT, parser.getWriterType());
+      wparams.set(CommonParams.VERSION, parser.getVersion());
+    }
+
+    //TODO add invariantParams support
+
+    String basePath = solrRequest.getBasePath() == null ? serverBaseUrl : solrRequest.getBasePath();
+    if (collection != null)
+      basePath += "/" + collection;
+
+    if (solrRequest instanceof V2Request) {
+      if (System.getProperty("solr.v2RealPath") == null) {
+        basePath = serverBaseUrl.replace("/solr", "/api");
+      } else {
+        basePath = serverBaseUrl + "/____v2";
+      }
+    }
+
+    if (SolrRequest.METHOD.GET == solrRequest.getMethod()) {
+      if (streams != null || contentWriter != null) {
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
+      }
+
+      return httpClient.newRequest(basePath + path + wparams.toQueryString()).method(HttpMethod.GET);
+    }
+
+    if (SolrRequest.METHOD.DELETE == solrRequest.getMethod()) {
+      return httpClient.newRequest(basePath + path + wparams.toQueryString()).method(HttpMethod.DELETE);
+    }
+
+    if (SolrRequest.METHOD.POST == solrRequest.getMethod() || SolrRequest.METHOD.PUT == solrRequest.getMethod()) {
+
+      String url = basePath + path;
+      boolean hasNullStreamName = false;
+      if (streams != null) {
+        hasNullStreamName = streams.stream().anyMatch(cs -> cs.getName() == null);
+      }
+
+      boolean isMultipart = streams != null && streams.size() > 1 && !hasNullStreamName;
+
+      HttpMethod method = SolrRequest.METHOD.POST == solrRequest.getMethod() ? HttpMethod.POST : HttpMethod.PUT;
+
+      if (contentWriter != null) {
+        Request req = httpClient
+            .newRequest(url + wparams.toQueryString())
+            .method(method);
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        contentWriter.write(baos);
+
+        //TODO reduce memory usage
+        return req.content(new BytesContentProvider(contentWriter.getContentType(), baos.toByteArray()));
+      } else if (streams == null || isMultipart) {
+        // send server list and request list as query string params
+        ModifiableSolrParams queryParams = calculateQueryParams(this.queryParams, wparams);
+        queryParams.add(calculateQueryParams(solrRequest.getQueryParams(), wparams));
+        Request req = httpClient
+            .newRequest(url + queryParams.toQueryString())
+            .method(method);
+        return fillContentStream(req, streams, wparams, isMultipart);
+      } else {
+        // It is has one stream, it is the post body, put the params in the URL
+        ContentStream contentStream = streams.iterator().next();
+        return httpClient
+            .newRequest(url + wparams.toQueryString())
+            .method(method)
+            .content(new InputStreamContentProvider(contentStream.getStream()), contentStream.getContentType());
+      }
+    }
+
+    throw new SolrServerException("Unsupported method: " + solrRequest.getMethod());
+  }
+
+  private Request fillContentStream(Request req, Collection<ContentStream> streams,
+                                    ModifiableSolrParams wparams,
+                                    boolean isMultipart) throws IOException {
+    if (isMultipart) {
+      // multipart/form-data
+      MultiPartContentProvider content = new MultiPartContentProvider();
+      Iterator<String> iter = wparams.getParameterNamesIterator();
+      while (iter.hasNext()) {
+        String key = iter.next();
+        String[] vals = wparams.getParams(key);
+        if (vals != null) {
+          for (String val : vals) {
+            content.addFieldPart(key, new StringContentProvider(val), null);
+          }
+        }
+      }
+      if (streams != null) {
+        for (ContentStream contentStream : streams) {
+          String contentType = contentStream.getContentType();
+          if (contentType == null) {
+            contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; // default
+          }
+          String name = contentStream.getName();
+          if (name == null) {
+            name = "";
+          }
+          HttpFields fields = new HttpFields();
+          fields.add(HttpHeader.CONTENT_TYPE, contentType);
+          content.addFilePart(name, contentStream.getName(), new InputStreamContentProvider(contentStream.getStream()), fields);
+        }
+      }
+      req.content(content);
+    } else {
+      // application/x-www-form-urlencoded
+      Fields fields = new Fields();
+      Iterator<String> iter = wparams.getParameterNamesIterator();
+      while (iter.hasNext()) {
+        String key = iter.next();
+        String[] vals = wparams.getParams(key);
+        if (vals != null) {
+          for (String val : vals) {
+            fields.add(key, val);
+          }
+        }
+      }
+      req.content(new FormContentProvider(fields, StandardCharsets.UTF_8));
+    }
+
+    return req;
+  }
+
+  private boolean wantStream(final ResponseParser processor) {
+    return processor == null || processor instanceof InputStreamResponseParser;
+  }
+
+  private NamedList<Object> processErrorsAndResponse(Response response,
+                                                     final ResponseParser processor,
+                                                     InputStream is,
+                                                     String encoding,
+                                                     final boolean isV2Api)
+      throws SolrServerException {
+    boolean shouldClose = true;
+    try {
+      // handle some http level checks before trying to parse the response
+      int httpStatus = response.getStatus();
+
+      String contentType;
+      contentType = response.getHeaders().get("content-type");
+      if (contentType == null) contentType = "";
+
+      switch (httpStatus) {
+        case HttpStatus.SC_OK:
+        case HttpStatus.SC_BAD_REQUEST:
+        case HttpStatus.SC_CONFLICT: // 409
+          break;
+        case HttpStatus.SC_MOVED_PERMANENTLY:
+        case HttpStatus.SC_MOVED_TEMPORARILY:
+          if (!httpClient.isFollowRedirects()) {
+            throw new SolrServerException("Server at " + getBaseURL()
+                + " sent back a redirect (" + httpStatus + ").");
+          }
+          break;
+        default:
+          if (processor == null || "".equals(contentType)) {
+            throw new RemoteSolrException(serverBaseUrl, httpStatus, "non ok status: " + httpStatus
+                + ", message:" + response.getReason(),
+                null);
+          }
+      }
+
+      if (wantStream(parser)) {
+        // no processor specified, return raw stream
+        NamedList<Object> rsp = new NamedList<>();
+        rsp.add("stream", is);
+        // Only case where stream should not be closed
+        shouldClose = false;
+        return rsp;
+      }
+
+      String procCt = processor.getContentType();
+      if (procCt != null) {
+        String procMimeType = ContentType.parse(procCt).getMimeType().trim().toLowerCase(Locale.ROOT);
+        String mimeType = ContentType.parse(contentType).getMimeType().trim().toLowerCase(Locale.ROOT);
+        if (!procMimeType.equals(mimeType)) {
+          // unexpected mime type
+          String msg = "Expected mime type " + procMimeType + " but got " + mimeType + ".";
+          try {
+            msg = msg + " " + IOUtils.toString(is, encoding);
+          } catch (IOException e) {
+            throw new RemoteSolrException(serverBaseUrl, httpStatus, "Could not parse response with encoding " + encoding, e);
+          }
+          throw new RemoteSolrException(serverBaseUrl, httpStatus, msg, null);
+        }
+      }
+
+      NamedList<Object> rsp;
+      try {
+        rsp = processor.processResponse(is, encoding);
+      } catch (Exception e) {
+        throw new RemoteSolrException(serverBaseUrl, httpStatus, e.getMessage(), e);
+      }
+
+      Object error = rsp == null ? null : rsp.get("error");
+      if (error != null && (String.valueOf(getObjectByPath(error, true, errPath)).endsWith("ExceptionWithErrObject"))) {
+        throw RemoteExecutionException.create(serverBaseUrl, rsp);
+      }
+      if (httpStatus != HttpStatus.SC_OK && !isV2Api) {
+        NamedList<String> metadata = null;
+        String reason = null;
+        try {
+          NamedList err = (NamedList) rsp.get("error");
+          if (err != null) {
+            reason = (String) err.get("msg");
+            if (reason == null) {
+              reason = (String) err.get("trace");
+            }
+            metadata = (NamedList<String>) err.get("metadata");
+          }
+        } catch (Exception ex) {}
+        if (reason == null) {
+          StringBuilder msg = new StringBuilder();
+          msg.append(response.getReason())
+              .append("\n\n")
+              .append("request: ")
+              .append(response.getRequest().getMethod());
+          try {
+            reason = java.net.URLDecoder.decode(msg.toString(), UTF_8);
+          } catch (UnsupportedEncodingException e) {
+          }
+        }
+        RemoteSolrException rss = new RemoteSolrException(serverBaseUrl, httpStatus, reason, null);
+        if (metadata != null) rss.setMetadata(metadata);
+        throw rss;
+      }
+      return rsp;
+    } finally {
+      if (shouldClose) {
+        try {
+          is.close();
+          assert ObjectReleaseTracker.release(is);
+        } catch (IOException e) {
+          // quitely
+        }
+      }
+    }
+  }
+
+  @Override
+  public NamedList<Object> request(SolrRequest request, String collection) throws SolrServerException, IOException {
+    return request(request, collection, null);
+  }
+
+  public void setRequestWriter(RequestWriter requestWriter) {
+    this.requestWriter = requestWriter;
+  }
+
+  public interface OnComplete {
+    void onSuccess(NamedList<Object> result);
+
+    void onFailure(Throwable e);
+  }
+
+  public void setFollowRedirects(boolean follow) {
+    httpClient.setFollowRedirects(follow);
+  }
+
+  public String getBaseURL() {
+    return serverBaseUrl;
+  }
+
+  private static class AsyncTracker {
+    private static final int MAX_OUTSTANDING_REQUESTS = 1000;
+
+    // wait for async requests
+    private final Phaser phaser;
+    // maximum outstanding requests left
+    private final Semaphore available;
+    private final Request.QueuedListener queuedListener;
+    private final Response.CompleteListener completeListener;
+
+    AsyncTracker() {
+      // TODO: what about shared instances?
+      phaser = new Phaser(1);
+      available = new Semaphore(MAX_OUTSTANDING_REQUESTS, false);
+      queuedListener = request -> {
+        phaser.register();
+        try {
+          available.acquire();
+        } catch (InterruptedException ignored) {
+
+        }
+      };
+      completeListener = result -> {
+        phaser.arriveAndDeregister();
+        available.release();
+      };
+    }
+
+    int getMaxRequestsQueuedPerDestination() {
+      // comfortably above max outstanding requests
+      return MAX_OUTSTANDING_REQUESTS * 3;
+    }
+
+    public void waitForComplete() {
+      phaser.arriveAndAwaitAdvance();
+      phaser.arriveAndDeregister();
+    }
+  }
+
+  public static class Builder {
+
+    private HttpClient httpClient;
+    private SSLConfig sslConfig = defaultSSLConfig;
+    private Integer idleTimeout;
+    private Integer connectionTimeout;
+    private Integer maxConnectionsPerHost;
+    private boolean useHttp1_1 = Boolean.getBoolean("solr.http1");
+    protected String baseSolrUrl;
+
+    public Builder() {
+
+    }
+
+    public Builder(String baseSolrUrl) {
+      this.baseSolrUrl = baseSolrUrl;
+    }
+
+    public Http2SolrClient build() {
+      return new Http2SolrClient(baseSolrUrl, this);
+    }
+
+    public Builder withHttpClient(HttpClient httpClient) {
+      this.httpClient = httpClient;
+      return this;
+    }
+
+    public Builder withSSLConfig(SSLConfig sslConfig) {
+      this.sslConfig = sslConfig;
+      return this;
+    }
+
+    /**
+     * Set maxConnectionsPerHost for http1 connections, maximum number http2 connections is limited by 4
+     */
+    public Builder maxConnectionsPerHost(int max) {
+      this.maxConnectionsPerHost = max;
+      return this;
+    }
+
+    public Builder idleTimeout(int idleConnectionTimeout) {
+      this.idleTimeout = idleConnectionTimeout;
+      return this;
+    }
+
+    public Builder useHttp1_1(boolean useHttp1_1) {
+      this.useHttp1_1 = useHttp1_1;
+      return this;
+    }
+
+    public Builder connectionTimeout(int connectionTimeOut) {
+      this.connectionTimeout = connectionTimeOut;
+      return this;
+    }
+
+  }
+
+  /**
+   * Subclass of SolrException that allows us to capture an arbitrary HTTP status code that may have been returned by
+   * the remote server or a proxy along the way.
+   */
+  public static class RemoteSolrException extends SolrException {
+    /**
+     * @param remoteHost the host the error was received from
+     * @param code       Arbitrary HTTP status code
+     * @param msg        Exception Message
+     * @param th         Throwable to wrap with this Exception
+     */
+    public RemoteSolrException(String remoteHost, int code, String msg, Throwable th) {
+      super(code, "Error from server at " + remoteHost + ": " + msg, th);
+    }
+  }
+
+  /**
+   * This should be thrown when a server has an error in executing the request and it sends a proper payload back to the
+   * client
+   */
+  public static class RemoteExecutionException extends RemoteSolrException {
+    private NamedList meta;
+
+    public RemoteExecutionException(String remoteHost, int code, String msg, NamedList meta) {
+      super(remoteHost, code, msg, null);
+      this.meta = meta;
+    }
+
+    public static RemoteExecutionException create(String host, NamedList errResponse) {
+      Object errObj = errResponse.get("error");
+      if (errObj != null) {
+        Number code = (Number) getObjectByPath(errObj, true, Collections.singletonList("code"));
+        String msg = (String) getObjectByPath(errObj, true, Collections.singletonList("msg"));
+        return new RemoteExecutionException(host, code == null ? ErrorCode.UNKNOWN.code : code.intValue(),
+            msg == null ? "Unknown Error" : msg, errResponse);
+
+      } else {
+        throw new RuntimeException("No error");
+      }
+    }
+
+    public NamedList getMetaData() {
+      return meta;
+    }
+  }
+
+  public Set<String> getQueryParams() {
+    return queryParams;
+  }
+
+  /**
+   * Expert Method
+   *
+   * @param queryParams set of param keys to only send via the query string
+   *                    Note that the param will be sent as a query string if the key is part
+   *                    of this Set or the SolrRequest's query params.
+   * @see org.apache.solr.client.solrj.SolrRequest#getQueryParams
+   */
+  public void setQueryParams(Set<String> queryParams) {
+    this.queryParams = queryParams;
+  }
+
+  private ModifiableSolrParams calculateQueryParams(Set<String> queryParamNames,
+                                                    ModifiableSolrParams wparams) {
+    ModifiableSolrParams queryModParams = new ModifiableSolrParams();
+    if (queryParamNames != null) {
+      for (String param : queryParamNames) {
+        String[] value = wparams.getParams(param);
+        if (value != null) {
+          for (String v : value) {
+            queryModParams.add(param, v);
+          }
+          wparams.remove(param);
+        }
+      }
+    }
+    return queryModParams;
+  }
+
+  public ResponseParser getParser() {
+    return parser;
+  }
+
+  public void setParser(ResponseParser processor) {
+    parser = processor;
+  }
+
+  public static void setDefaultSSLConfig(SSLConfig sslConfig) {
+    Http2SolrClient.defaultSSLConfig = sslConfig;
+  }
+
+  // public for testing, only used by tests
+  public static void resetSslContextFactory() {
+    Http2SolrClient.defaultSSLConfig = null;
+  }
+
+  private static SslContextFactory getDefaultSslContextFactory() {
+    SslContextFactory sslContextFactory = new SslContextFactory(false);
+
+    if (null != System.getProperty("javax.net.ssl.keyStore")) {
+      sslContextFactory.setKeyStorePath
+          (System.getProperty("javax.net.ssl.keyStore"));
+    }
+    if (null != System.getProperty("javax.net.ssl.keyStorePassword")) {
+      sslContextFactory.setKeyStorePassword
+          (System.getProperty("javax.net.ssl.keyStorePassword"));
+    }
+    if (null != System.getProperty("javax.net.ssl.trustStore")) {
+      sslContextFactory.setTrustStorePath
+          (System.getProperty("javax.net.ssl.trustStore"));
+    }
+    if (null != System.getProperty("javax.net.ssl.trustStorePassword")) {
+      sslContextFactory.setTrustStorePassword
+          (System.getProperty("javax.net.ssl.trustStorePassword"));
+    }
+
+    String checkPeerNameStr = System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME);
+    boolean sslCheckPeerName = true;
+    if (checkPeerNameStr == null || "false".equalsIgnoreCase(checkPeerNameStr)) {
+      sslCheckPeerName = false;
+    }
+
+    if (System.getProperty("tests.jettySsl.clientAuth") != null) {
+      sslCheckPeerName = sslCheckPeerName || Boolean.getBoolean("tests.jettySsl.clientAuth");
+    }
+
+    sslContextFactory.setNeedClientAuth(sslCheckPeerName);
+    return sslContextFactory;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f80e8e11/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientBuilderFactory.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientBuilderFactory.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientBuilderFactory.java
index 77c4a94..b552f66 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientBuilderFactory.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientBuilderFactory.java
@@ -38,4 +38,7 @@ public interface HttpClientBuilderFactory extends Closeable {
    */
   public SolrHttpClientBuilder getHttpClientBuilder(Optional<SolrHttpClientBuilder> builder);
 
+  public default void setup(Http2SolrClient client) {
+
+  }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f80e8e11/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpListenerFactory.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpListenerFactory.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpListenerFactory.java
new file mode 100644
index 0000000..41b58ae
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpListenerFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.api.Result;
+
+public interface HttpListenerFactory {
+  abstract class RequestResponseListener implements Request.BeginListener, Response.CompleteListener, Request.QueuedListener {
+    @Override
+    public void onBegin(Request request){}
+
+    @Override
+    public void onQueued(Request request) {}
+
+    @Override
+    public void onComplete(Result result) {}
+  }
+
+  RequestResponseListener get();
+}
+

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f80e8e11/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
index afa2ef6..51977d7 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
@@ -16,17 +16,19 @@
  */
 package org.apache.solr.client.solrj.impl;
 
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.Configuration;
 import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.nio.file.Paths;
 import java.security.Principal;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 
-import javax.security.auth.login.AppConfigurationEntry;
-import javax.security.auth.login.Configuration;
-
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpRequestInterceptor;
@@ -41,6 +43,9 @@ import org.apache.http.cookie.CookieSpecProvider;
 import org.apache.http.entity.BufferedHttpEntity;
 import org.apache.http.impl.auth.SPNegoSchemeFactory;
 import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.eclipse.jetty.client.HttpAuthenticationStore;
+import org.eclipse.jetty.client.WWWAuthenticationProtocolHandler;
+import org.eclipse.jetty.client.util.SPNEGOAuthentication;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -50,8 +55,9 @@ import org.slf4j.LoggerFactory;
 public class Krb5HttpClientBuilder implements HttpClientBuilderFactory {
   
   public static final String LOGIN_CONFIG_PROP = "java.security.auth.login.config";
+  private static final String SPNEGO_OID = "1.3.6.1.5.5.2";
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-  
+
   private static Configuration jaasConfig = new SolrJaasConfiguration();
 
   public Krb5HttpClientBuilder() {
@@ -80,6 +86,49 @@ public class Krb5HttpClientBuilder implements HttpClientBuilderFactory {
     return builder.isPresent() ? getBuilder(builder.get()) : getBuilder();
   }
 
+  private SPNEGOAuthentication createSPNEGOAuthentication() {
+    SPNEGOAuthentication authentication = new SPNEGOAuthentication(null){
+
+      public boolean matches(String type, URI uri, String realm) {
+        return this.getType().equals(type);
+      }
+    };
+    String clientAppName = System.getProperty("solr.kerberos.jaas.appname", "Client");
+    AppConfigurationEntry[] entries = jaasConfig.getAppConfigurationEntry(clientAppName);
+    if (entries == null) {
+      log.warn("Could not find login configuration entry for {}. SPNego authentication may not be successful.", (Object)clientAppName);
+      return authentication;
+    }
+    if (entries.length != 1) {
+      log.warn("Multiple login modules are specified in the configuration file");
+      return authentication;
+    }
+    Map<String, ?> options = entries[0].getOptions();
+    String keyTab = (String)options.get("keyTab");
+    if (keyTab != null) {
+      authentication.setUserKeyTabPath(Paths.get(keyTab, new String[0]));
+    }
+    authentication.setServiceName("HTTP");
+    authentication.setUserName((String)options.get("principal"));
+    if ("true".equalsIgnoreCase((String)options.get("useTicketCache"))) {
+      authentication.setUseTicketCache(true);
+      String ticketCachePath = (String)options.get("ticketCache");
+      if (ticketCachePath != null) {
+        authentication.setTicketCachePath(Paths.get(ticketCachePath));
+      }
+      authentication.setRenewTGT("true".equalsIgnoreCase((String)options.get("renewTGT")));
+    }
+    return authentication;
+  }
+
+  @Override
+  public void setup(Http2SolrClient http2Client) {
+    HttpAuthenticationStore authenticationStore = new HttpAuthenticationStore();
+    authenticationStore.addAuthentication(createSPNEGOAuthentication());
+    http2Client.getHttpClient().setAuthenticationStore(authenticationStore);
+    http2Client.getProtocolHandlers().put(new WWWAuthenticationProtocolHandler(http2Client.getHttpClient()));
+  }
+
   public SolrHttpClientBuilder getBuilder(SolrHttpClientBuilder builder) {
     if (System.getProperty(LOGIN_CONFIG_PROP) != null) {
       String configValue = System.getProperty(LOGIN_CONFIG_PROP);
@@ -93,8 +142,7 @@ public class Krb5HttpClientBuilder implements HttpClientBuilderFactory {
         // authentication mechanism can load the credentials from the JAAS configuration.
         if (useSubjectCredsVal == null) {
           System.setProperty(useSubjectCredsProp, "false");
-        }
-        else if (!useSubjectCredsVal.toLowerCase(Locale.ROOT).equals("false")) {
+        } else if (!useSubjectCredsVal.toLowerCase(Locale.ROOT).equals("false")) {
           // Don't overwrite the prop value if it's already been written to something else,
           // but log because it is likely the Credentials won't be loaded correctly.
           log.warn("System Property: " + useSubjectCredsProp + " set to: " + useSubjectCredsVal

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f80e8e11/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java
new file mode 100644
index 0000000..293a264
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import java.util.Arrays;
+
+import org.apache.solr.client.solrj.SolrClient;
+
+/**
+ * LBHttp2SolrClient or "LoadBalanced LBHttp2SolrClient" is a load balancing wrapper around
+ * {@link Http2SolrClient}. This is useful when you
+ * have multiple Solr servers and the requests need to be Load Balanced among them.
+ *
+ * Do <b>NOT</b> use this class for indexing in master/slave scenarios since documents must be sent to the
+ * correct master; no inter-node routing is done.
+ *
+ * In SolrCloud (leader/replica) scenarios, it is usually better to use
+ * {@link CloudSolrClient}, but this class may be used
+ * for updates because the server will forward them to the appropriate leader.
+ *
+ * <p>
+ * It offers automatic failover when a server goes down and it detects when the server comes back up.
+ * <p>
+ * Load balancing is done using a simple round-robin on the list of servers.
+ * <p>
+ * If a request to a server fails by an IOException due to a connection timeout or read timeout then the host is taken
+ * off the list of live servers and moved to a 'dead server list' and the request is resent to the next live server.
+ * This process is continued till it tries all the live servers. If at least one server is alive, the request succeeds,
+ * and if not it fails.
+ * <blockquote><pre>
+ * SolrClient lbHttp2SolrClient = new LBHttp2SolrClient(http2SolrClient, "http://host1:8080/solr/", "http://host2:8080/solr", "http://host2:8080/solr");
+ * </pre></blockquote>
+ * This detects if a dead server comes alive automatically. The check is done in fixed intervals in a dedicated thread.
+ * This interval can be set using {@link #setAliveCheckInterval} , the default is set to one minute.
+ * <p>
+ * <b>When to use this?</b><br> This can be used as a software load balancer when you do not wish to setup an external
+ * load balancer. Alternatives to this code are to use
+ * a dedicated hardware load balancer or using Apache httpd with mod_proxy_balancer as a load balancer. See <a
+ * href="http://en.wikipedia.org/wiki/Load_balancing_(computing)">Load balancing on Wikipedia</a>
+ *
+ * @lucene.experimental
+ * @since solr 8.0
+ */
+public class LBHttp2SolrClient extends LBSolrClient {
+  private Http2SolrClient httpClient;
+
+  public LBHttp2SolrClient(Http2SolrClient httpClient, String... baseSolrUrls) {
+    super(Arrays.asList(baseSolrUrls));
+    this.httpClient = httpClient;
+  }
+  @Override
+  protected SolrClient getClient(String baseUrl) {
+    return httpClient;
+  }
+}