You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ja...@apache.org on 2023/11/25 00:48:38 UTC
(solr) branch branch_9x updated: SOLR-16974: Global Circuit Breakers (#1919) (#2070)
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new f5f54429b34 SOLR-16974: Global Circuit Breakers (#1919) (#2070)
f5f54429b34 is described below
commit f5f54429b34fd71af06b7331cfda070ea56a1580
Author: Jan Høydahl <ja...@apache.org>
AuthorDate: Sat Nov 25 01:48:33 2023 +0100
SOLR-16974: Global Circuit Breakers (#1919) (#2070)
Co-authored-by: Christine Poerschke <cp...@apache.org>
(cherry picked from commit e740123546c7f20c6728df42d493e11f93dc8c79)
---
solr/bin/solr | 22 +++
.../src/java/org/apache/solr/core/SolrCore.java | 6 +-
.../org/apache/solr/core/SolrResourceLoader.java | 2 +
.../util/circuitbreaker/CPUCircuitBreaker.java | 61 +++---
.../circuitbreaker/CircuitBreakerRegistry.java | 204 +++++++++++++++++----
.../circuitbreaker/LoadAverageCircuitBreaker.java | 3 +-
.../util/circuitbreaker/MemoryCircuitBreaker.java | 3 +-
.../apache/solr/util/BaseTestCircuitBreaker.java | 19 +-
.../apache/solr/util/TestGlobalCircuitBreaker.java | 73 ++++++++
9 files changed, 323 insertions(+), 70 deletions(-)
diff --git a/solr/bin/solr b/solr/bin/solr
old mode 100644
new mode 100755
index f3356343e9f..5a66d7ff60e
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -1882,6 +1882,28 @@ if [ "${SOLR_ENABLE_STREAM_BODY:-false}" == "true" ]; then
SCRIPT_SOLR_OPTS+=("-Dsolr.enableStreamBody=true")
fi
+# Parse global circuit breaker env vars and convert to dot separated, lowercase properties
+if [ -n "${SOLR_CIRCUITBREAKER_UPDATE_CPU:-}" ]; then
+ SOLR_OPTS+=("-Dsolr.circuitbreaker.update.cpu=$SOLR_CIRCUITBREAKER_UPDATE_CPU")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_UPDATE_MEM:-}" ]; then
+ SOLR_OPTS+=("-Dsolr.circuitbreaker.update.mem=$SOLR_CIRCUITBREAKER_UPDATE_MEM")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_UPDATE_LOADAVG:-}" ]; then
+ SOLR_OPTS+=("-Dsolr.circuitbreaker.update.loadavg=$SOLR_CIRCUITBREAKER_UPDATE_LOADAVG")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_QUERY_CPU:-}" ]; then
+ SOLR_OPTS+=("-Dsolr.circuitbreaker.query.cpu=$SOLR_CIRCUITBREAKER_QUERY_CPU")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_QUERY_MEM:-}" ]; then
+ SOLR_OPTS+=("-Dsolr.circuitbreaker.query.mem=$SOLR_CIRCUITBREAKER_QUERY_MEM")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_QUERY_LOADAVG:-}" ]; then
+ SOLR_OPTS+=("-Dsolr.circuitbreaker.query.loadavg=$SOLR_CIRCUITBREAKER_QUERY_LOADAVG")
+fi
+
+echo "SOLR_OPTS is now: ${SOLR_OPTS[*]}"
+
: ${SOLR_SERVER_DIR:=$DEFAULT_SERVER_DIR}
if [ ! -e "$SOLR_SERVER_DIR" ]; then
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 0670ce2a775..094ccb285fc 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -246,7 +246,7 @@ public class SolrCore implements SolrInfoBean, Closeable {
private final ConfigSet configSet;
// singleton listener for all packages used in schema
- private final CircuitBreakerRegistry circuitBreakerRegistry = new CircuitBreakerRegistry();
+ private final CircuitBreakerRegistry circuitBreakerRegistry;
private final List<Runnable> confListeners = new CopyOnWriteArrayList<>();
@@ -1072,6 +1072,7 @@ public class SolrCore implements SolrInfoBean, Closeable {
final CountDownLatch latch = new CountDownLatch(1);
try {
this.coreContainer = coreContainer;
+ this.circuitBreakerRegistry = new CircuitBreakerRegistry(coreContainer);
this.configSet = configSet;
this.coreDescriptor = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null");
this.name = Objects.requireNonNull(coreDescriptor.getName());
@@ -3173,6 +3174,9 @@ public class SolrCore implements SolrInfoBean, Closeable {
type.getSimpleName() + "." + info.name, (SolrMetricProducer) o);
}
if (o instanceof CircuitBreaker) {
+ if (o instanceof SolrCoreAware) {
+ ((SolrCoreAware) o).inform(this);
+ }
circuitBreakerRegistry.register((CircuitBreaker) o);
}
if (info.isDefault()) {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 438682fdd43..898f2801304 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -72,6 +72,7 @@ import org.apache.solr.schema.ManagedIndexSchemaFactory;
import org.apache.solr.schema.SimilarityFactory;
import org.apache.solr.search.QParserPlugin;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
+import org.apache.solr.util.circuitbreaker.CircuitBreaker;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -814,6 +815,7 @@ public class SolrResourceLoader
new Class<?>[] {
// DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
// VIA RUNTIME APIS -- UNTIL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
+ CircuitBreaker.class,
CodecFactory.class,
DirectoryFactory.class,
ManagedIndexSchemaFactory.class,
diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java
index 4c1ac111c58..72b91f8722c 100644
--- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java
+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java
@@ -20,9 +20,10 @@ package org.apache.solr.util.circuitbreaker;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Metric;
import java.lang.invoke.MethodHandles;
-import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,35 +34,30 @@ import org.slf4j.LoggerFactory;
* We depend on OperatingSystemMXBean which does not allow a configurable interval of collection of
* data.
*/
-public class CPUCircuitBreaker extends CircuitBreaker {
+public class CPUCircuitBreaker extends CircuitBreaker implements SolrCoreAware {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private boolean enabled = true;
+ private boolean enabled = false;
private double cpuUsageThreshold;
- private final SolrCore core;
+ private CoreContainer cc;
private static final ThreadLocal<Double> seenCPUUsage = ThreadLocal.withInitial(() -> 0.0);
private static final ThreadLocal<Double> allowedCPUUsage = ThreadLocal.withInitial(() -> 0.0);
- public CPUCircuitBreaker(SolrCore core) {
+ public CPUCircuitBreaker() {
super();
- this.core = core;
}
- @Override
- public void init(NamedList<?> args) {
- super.init(args);
- double localSeenCPUUsage = calculateLiveCPUUsage();
+ @Deprecated(since = "9.5")
+ public CPUCircuitBreaker(SolrCore core) {
+ this(core.getCoreContainer());
+ }
- if (localSeenCPUUsage < 0) {
- String msg =
- "Initialization failure for CPU circuit breaker. Unable to get 'systemCpuLoad', not supported by the JVM?";
- if (log.isErrorEnabled()) {
- log.error(msg);
- }
- enabled = false;
- }
+ public CPUCircuitBreaker(CoreContainer coreContainer) {
+ super();
+ this.cc = coreContainer;
+ enableIfSupported();
}
@Override
@@ -91,7 +87,7 @@ public class CPUCircuitBreaker extends CircuitBreaker {
+ allowedCPUUsage.get();
}
- public void setThreshold(double thresholdValueInPercentage) {
+ public CPUCircuitBreaker setThreshold(double thresholdValueInPercentage) {
if (thresholdValueInPercentage > 100) {
throw new IllegalArgumentException("Invalid Invalid threshold value.");
}
@@ -100,6 +96,7 @@ public class CPUCircuitBreaker extends CircuitBreaker {
throw new IllegalStateException("Threshold cannot be less than or equal to zero");
}
cpuUsageThreshold = thresholdValueInPercentage;
+ return this;
}
public double getCpuUsageThreshold() {
@@ -114,12 +111,7 @@ public class CPUCircuitBreaker extends CircuitBreaker {
protected double calculateLiveCPUUsage() {
// TODO: Use Codahale Meter to calculate the value
Metric metric =
- this.core
- .getCoreContainer()
- .getMetricManager()
- .registry("solr.jvm")
- .getMetrics()
- .get("os.systemCpuLoad");
+ this.cc.getMetricManager().registry("solr.jvm").getMetrics().get("os.systemCpuLoad");
if (metric == null) {
return -1.0;
@@ -137,4 +129,23 @@ public class CPUCircuitBreaker extends CircuitBreaker {
return -1.0; // Unable to unpack metric
}
+
+ @Override
+ public void inform(SolrCore core) {
+ this.cc = core.getCoreContainer();
+ enableIfSupported();
+ }
+
+ private void enableIfSupported() {
+ if (calculateLiveCPUUsage() < 0) {
+ String msg =
+ "Initialization failure for CPU circuit breaker. Unable to get 'systemCpuLoad', not supported by the JVM?";
+ if (log.isErrorEnabled()) {
+ log.error(msg);
+ }
+ enabled = false;
+ } else {
+ enabled = true;
+ }
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java
index a7081df96f6..14e9ee2bb47 100644
--- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java
+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java
@@ -22,30 +22,100 @@ import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import org.apache.solr.client.solrj.SolrRequest.SolrRequestType;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.core.CoreContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Keeps track of all registered circuit breaker instances for various request types. Responsible
- * for a holistic view of whether a circuit breaker has tripped or not.
+ * for a holistic view of whether a circuit breaker has tripped or not. Circuit breakers may be
+ * registered globally and/or per-core. This registry has one instance per core, but keeps a static
+ * map of globally registered Circuit Breakers that are always checked.
*
* @lucene.experimental
* @since 9.4
*/
public class CircuitBreakerRegistry implements Closeable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
private final Map<SolrRequestType, List<CircuitBreaker>> circuitBreakerMap = new HashMap<>();
+ private static final Map<SolrRequestType, List<CircuitBreaker>> globalCircuitBreakerMap =
+ new HashMap<>();
+ private static final Pattern SYSPROP_REGEX =
+ Pattern.compile("solr.circuitbreaker\\.(update|query)\\.(cpu|mem|loadavg)");
+ public static final String SYSPROP_PREFIX = "solr.circuitbreaker.";
+ public static final String SYSPROP_UPDATE_CPU = SYSPROP_PREFIX + "update.cpu";
+ public static final String SYSPROP_UPDATE_MEM = SYSPROP_PREFIX + "update.mem";
+ public static final String SYSPROP_UPDATE_LOADAVG = SYSPROP_PREFIX + "update.loadavg";
+ public static final String SYSPROP_QUERY_CPU = SYSPROP_PREFIX + "query.cpu";
+ public static final String SYSPROP_QUERY_MEM = SYSPROP_PREFIX + "query.mem";
+ public static final String SYSPROP_QUERY_LOADAVG = SYSPROP_PREFIX + "query.loadavg";
+
+ public CircuitBreakerRegistry(CoreContainer coreContainer) {
+ initGlobal(coreContainer);
+ }
- public CircuitBreakerRegistry() {}
+ private static void initGlobal(CoreContainer coreContainer) {
+ // Read system properties to register global circuit breakers for update and query:
+ // Example: solr.circuitbreaker.update.cpu = 50
+ System.getProperties().keySet().stream()
+ .map(k -> SYSPROP_REGEX.matcher(k.toString()))
+ .filter(Matcher::matches)
+ .collect(Collectors.groupingBy(m -> m.group(2) + ":" + System.getProperty(m.group(0))))
+ .forEach(
+ (breakerAndValue, breakers) -> {
+ CircuitBreaker breaker;
+ String[] breakerAndValueArr = breakerAndValue.split(":");
+ switch (breakerAndValueArr[0]) {
+ case "cpu":
+ breaker =
+ new CPUCircuitBreaker(coreContainer)
+ .setThreshold(Double.parseDouble(breakerAndValueArr[1]));
+ break;
+ case "mem":
+ breaker =
+ new MemoryCircuitBreaker()
+ .setThreshold(Double.parseDouble(breakerAndValueArr[1]));
+ break;
+ case "loadavg":
+ breaker =
+ new LoadAverageCircuitBreaker()
+ .setThreshold(Double.parseDouble(breakerAndValueArr[1]));
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unknown circuit breaker type: " + breakerAndValueArr[0]);
+ }
+ breaker.setRequestTypes(
+ breakers.stream().map(m -> m.group(1)).collect(Collectors.toList()));
+ registerGlobal(breaker);
+ if (log.isInfoEnabled()) {
+ log.info(
+ "Registered global circuit breaker {} for request type(s) {}",
+ breakerAndValue,
+ breaker.getRequestTypes());
+ }
+ });
+ }
+ /** List all registered circuit breakers for global context */
+ public static Set<CircuitBreaker> listGlobal() {
+ return globalCircuitBreakerMap.values().stream()
+ .flatMap(List::stream)
+ .collect(Collectors.toSet());
+ }
+
+ /** Register a circuit breaker for a core */
public void register(CircuitBreaker circuitBreaker) {
synchronized (circuitBreakerMap) {
circuitBreaker
@@ -65,9 +135,27 @@ public class CircuitBreakerRegistry implements Closeable {
}
}
+ /** Register a global circuit breaker */
+ public static void registerGlobal(CircuitBreaker circuitBreaker) {
+ circuitBreaker
+ .getRequestTypes()
+ .forEach(
+ r -> {
+ List<CircuitBreaker> list =
+ globalCircuitBreakerMap.computeIfAbsent(r, k -> new ArrayList<>());
+ list.add(circuitBreaker);
+ });
+ }
+
@VisibleForTesting
public void deregisterAll() throws IOException {
this.close();
+ deregisterGlobal();
+ }
+
+ @VisibleForTesting
+ public static void deregisterGlobal() {
+ closeGlobal();
}
/**
@@ -77,10 +165,10 @@ public class CircuitBreakerRegistry implements Closeable {
* @return CircuitBreakers which have triggered, null otherwise.
*/
public List<CircuitBreaker> checkTripped(SolrRequestType requestType) {
+ Map<SolrRequestType, List<CircuitBreaker>> combinedMap = getCombinedMap();
+ final List<CircuitBreaker> breakersOfType = combinedMap.get(requestType);
List<CircuitBreaker> triggeredCircuitBreakers = null;
-
- for (CircuitBreaker circuitBreaker :
- circuitBreakerMap.getOrDefault(requestType, Collections.emptyList())) {
+ for (CircuitBreaker circuitBreaker : breakersOfType) {
if (circuitBreaker.isTripped()) {
if (triggeredCircuitBreakers == null) {
triggeredCircuitBreakers = new ArrayList<>();
@@ -111,43 +199,83 @@ public class CircuitBreakerRegistry implements Closeable {
}
public boolean isEnabled(SolrRequestType requestType) {
- return circuitBreakerMap.containsKey(requestType);
+ return circuitBreakerMap.containsKey(requestType)
+ || globalCircuitBreakerMap.containsKey(requestType);
}
@Override
public void close() throws IOException {
synchronized (circuitBreakerMap) {
- final AtomicInteger closeFailedCounter = new AtomicInteger(0);
- circuitBreakerMap
- .values()
- .forEach(
- list ->
- list.forEach(
- it -> {
- try {
- if (log.isDebugEnabled()) {
- log.debug(
- "Closed circuit breaker {} for request type(s) {}",
- it.getClass().getSimpleName(),
- it.getRequestTypes());
- }
- it.close();
- } catch (IOException e) {
- if (log.isErrorEnabled()) {
- log.error(
- String.format(
- Locale.ROOT,
- "Failed to close circuit breaker %s",
- it.getClass().getSimpleName()),
- e);
- }
- closeFailedCounter.incrementAndGet();
- }
- }));
+ closeCircuitBreakers(
+ circuitBreakerMap.values().stream().flatMap(List::stream).collect(Collectors.toList()));
circuitBreakerMap.clear();
- if (closeFailedCounter.get() > 0) {
- throw new IOException("Failed to close " + closeFailedCounter.get() + " circuit breakers");
- }
}
}
+
+ private static void closeGlobal() {
+ synchronized (globalCircuitBreakerMap) {
+ closeCircuitBreakers(
+ globalCircuitBreakerMap.values().stream()
+ .flatMap(List::stream)
+ .collect(Collectors.toList()));
+ globalCircuitBreakerMap.clear();
+ }
+ }
+
+ /**
+ * Close a list of circuit breakers, tracing any failures.
+ *
+ * @throws SolrException if any CB fails to close
+ */
+ private static void closeCircuitBreakers(List<CircuitBreaker> breakers) {
+ final AtomicInteger closeFailedCounter = new AtomicInteger(0);
+ breakers.forEach(
+ it -> {
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug(
+ "Closing circuit breaker {} for request type(s) {}",
+ it.getClass().getSimpleName(),
+ it.getRequestTypes());
+ }
+ it.close();
+ } catch (IOException e) {
+ if (log.isErrorEnabled()) {
+ log.error(
+ String.format(
+ Locale.ROOT,
+ "Failed to close circuit breaker %s for request type(s) %s",
+ it.getClass().getSimpleName(),
+ it.getRequestTypes()),
+ e);
+ }
+ closeFailedCounter.incrementAndGet();
+ }
+ });
+ if (closeFailedCounter.get() > 0) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Failed to close " + closeFailedCounter.get() + " circuit breakers");
+ }
+ }
+
+ /**
+ * Return a combined map of local and global circuit breaker maps, joining the two maps in a
+ * streaming fashion
+ */
+ private Map<SolrRequestType, List<CircuitBreaker>> getCombinedMap() {
+ Map<SolrRequestType, List<CircuitBreaker>> combinedMap = new HashMap<>(circuitBreakerMap);
+ globalCircuitBreakerMap.forEach(
+ (k, v) ->
+ combinedMap.merge(
+ k,
+ v,
+ (v1, v2) -> {
+ List<CircuitBreaker> newList = new ArrayList<>();
+ newList.addAll(v1);
+ newList.addAll(v2);
+ return newList;
+ }));
+ return combinedMap;
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/LoadAverageCircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/LoadAverageCircuitBreaker.java
index 77772b927b3..bceca970264 100644
--- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/LoadAverageCircuitBreaker.java
+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/LoadAverageCircuitBreaker.java
@@ -78,11 +78,12 @@ public class LoadAverageCircuitBreaker extends CircuitBreaker {
+ allowedLoadAverage.get();
}
- public void setThreshold(double thresholdValueUnbounded) {
+ public LoadAverageCircuitBreaker setThreshold(double thresholdValueUnbounded) {
if (thresholdValueUnbounded <= 0) {
throw new IllegalStateException("Threshold cannot be less than or equal to zero");
}
loadAverageThreshold = thresholdValueUnbounded;
+ return this;
}
public double getLoadAverageThreshold() {
diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java
index 4a3eb3f5b9f..65e352e5d72 100644
--- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java
+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java
@@ -77,7 +77,7 @@ public class MemoryCircuitBreaker extends CircuitBreaker {
}
}
- public void setThreshold(double thresholdValueInPercentage) {
+ public MemoryCircuitBreaker setThreshold(double thresholdValueInPercentage) {
long currentMaxHeap = MEMORY_MX_BEAN.getHeapMemoryUsage().getMax();
if (currentMaxHeap <= 0) {
@@ -90,6 +90,7 @@ public class MemoryCircuitBreaker extends CircuitBreaker {
if (heapMemoryThreshold <= 0) {
throw new IllegalStateException("Memory limit cannot be less than or equal to zero");
}
+ return this;
}
@Override
diff --git a/solr/core/src/test/org/apache/solr/util/BaseTestCircuitBreaker.java b/solr/core/src/test/org/apache/solr/util/BaseTestCircuitBreaker.java
index 14c83df771a..880d10d162a 100644
--- a/solr/core/src/test/org/apache/solr/util/BaseTestCircuitBreaker.java
+++ b/solr/core/src/test/org/apache/solr/util/BaseTestCircuitBreaker.java
@@ -31,10 +31,11 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.SolrNamedThreadFactory;
-import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.CoreContainer;
import org.apache.solr.util.circuitbreaker.CPUCircuitBreaker;
import org.apache.solr.util.circuitbreaker.CircuitBreaker;
import org.apache.solr.util.circuitbreaker.CircuitBreakerManager;
+import org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry;
import org.apache.solr.util.circuitbreaker.LoadAverageCircuitBreaker;
import org.apache.solr.util.circuitbreaker.MemoryCircuitBreaker;
import org.hamcrest.MatcherAssert;
@@ -81,6 +82,16 @@ public abstract class BaseTestCircuitBreaker extends SolrTestCaseJ4 {
h.getCore().getCircuitBreakerRegistry().register(circuitBreaker);
+ expectThrows(SolrException.class, () -> h.query(req("name:\"john smith\"")));
+ }
+
+ public void testGlobalCBAlwaysTrips() {
+ removeAllExistingCircuitBreakers();
+
+ CircuitBreaker circuitBreaker = new MockCircuitBreaker(true);
+
+ CircuitBreakerRegistry.registerGlobal(circuitBreaker);
+
expectThrows(
SolrException.class,
() -> {
@@ -135,7 +146,7 @@ public abstract class BaseTestCircuitBreaker extends SolrTestCaseJ4 {
}
public void testFakeCPUCircuitBreaker() {
- CPUCircuitBreaker circuitBreaker = new FakeCPUCircuitBreaker(h.getCore());
+ CPUCircuitBreaker circuitBreaker = new FakeCPUCircuitBreaker(h.getCore().getCoreContainer());
circuitBreaker.setThreshold(75);
assertThatHighQueryLoadTrips(circuitBreaker, 5);
@@ -318,8 +329,8 @@ public abstract class BaseTestCircuitBreaker extends SolrTestCaseJ4 {
}
private static class FakeCPUCircuitBreaker extends CPUCircuitBreaker {
- public FakeCPUCircuitBreaker(SolrCore core) {
- super(core);
+ public FakeCPUCircuitBreaker(CoreContainer coreContainer) {
+ super(coreContainer);
}
@Override
diff --git a/solr/core/src/test/org/apache/solr/util/TestGlobalCircuitBreaker.java b/solr/core/src/test/org/apache/solr/util/TestGlobalCircuitBreaker.java
new file mode 100644
index 00000000000..428ce6edf68
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/TestGlobalCircuitBreaker.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/** Tests the pluggable circuit breaker implementation. The actual tests are in base class. */
+public class TestGlobalCircuitBreaker extends SolrTestCaseJ4 {
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+ System.setProperty("filterCache.enabled", "false");
+ System.setProperty("queryResultCache.enabled", "false");
+ System.setProperty("documentCache.enabled", "true");
+
+ // Set a global update breaker for a low CPU, which will trip during indexing
+ System.setProperty(CircuitBreakerRegistry.SYSPROP_UPDATE_LOADAVG, "0.1");
+
+ initCore("solrconfig-basic.xml", "schema.xml");
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ System.clearProperty(CircuitBreakerRegistry.SYSPROP_UPDATE_LOADAVG);
+ // Deregister the global breaker to not interfere with other tests
+ CircuitBreakerRegistry.deregisterGlobal();
+ }
+
+ @Test
+ public void testGloalCbRegistered() {
+ assertEquals(1, CircuitBreakerRegistry.listGlobal().size());
+ }
+
+ @Test
+ public void testIndexingTripsCpuCb() {
+ try {
+ for (int i = 0; i < 100; i++) {
+ assertU(adoc("name", "john smith", "id", "1"));
+ assertU(adoc("name", "johathon smith", "id", "2"));
+ assertU(adoc("name", "john percival smith", "id", "3"));
+ assertU(adoc("id", "1", "title", "this is a title.", "inStock_b1", "true"));
+ assertU(adoc("id", "2", "title", "this is another title.", "inStock_b1", "true"));
+ assertU(adoc("id", "3", "title", "Mary had a little lamb.", "inStock_b1", "false"));
+
+ // commit inside the loop to get multiple segments to make search as realistic as possible
+ assertU(commit());
+ }
+ fail("Should have tripped");
+ } catch (SolrException e) {
+ // We get a load average above 0.1, which trips the breaker
+ assertEquals(SolrException.ErrorCode.TOO_MANY_REQUESTS.code, e.code());
+ }
+ }
+}