You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/05/07 19:02:10 UTC
[1/3] lucene-solr:jira/solr-11779: SOLR-11779: Initial patch.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-11779 [created] d778367b6
SOLR-11779: Initial patch.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/71f03617
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/71f03617
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/71f03617
Branch: refs/heads/jira/solr-11779
Commit: 71f0361726de82fd0da85728defe842efba3fcb7
Parents: 507c439
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Apr 18 21:28:43 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Apr 18 21:28:43 2018 +0200
----------------------------------------------------------------------
lucene/ivy-versions.properties | 2 +
solr/core/ivy.xml | 2 +
.../handler/admin/MetricsHistoryHandler.java | 37 +++
.../apache/solr/metrics/rrd/SolrRrdBackend.java | 88 ++++++
.../solr/metrics/rrd/SolrRrdBackendFactory.java | 282 +++++++++++++++++++
5 files changed, 411 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71f03617/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 14e7194..4c4734b 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -299,6 +299,8 @@ org.restlet.jee.version = 2.3.0
/org.restlet.jee/org.restlet = ${org.restlet.jee.version}
/org.restlet.jee/org.restlet.ext.servlet = ${org.restlet.jee.version}
+/org.rrd4j/rrd4j = 3.2
+
/org.simpleframework/simple-xml = 2.7.1
org.slf4j.version = 1.7.24
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71f03617/solr/core/ivy.xml
----------------------------------------------------------------------
diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml
index e47d5b8..ee6fe80 100644
--- a/solr/core/ivy.xml
+++ b/solr/core/ivy.xml
@@ -155,6 +155,8 @@
<dependency org="org.codehaus.janino" name="commons-compiler" rev="${/org.codehaus.janino/commons-compiler}" conf="compile"/>
<dependency org="com.google.protobuf" name="protobuf-java" rev="${/com.google.protobuf/protobuf-java}" conf="compile"/>
+ <dependency org="org.rrd4j" name="rrd4j" rev="${/org.rrd4j/rrd4j}" conf="compile"/>
+
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71f03617/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
new file mode 100644
index 0000000..cc372cf
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.admin;
+
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+
+/**
+ *
+ */
+public class MetricsHistoryHandler extends RequestHandlerBase {
+
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+
+ }
+
+ @Override
+ public String getDescription() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71f03617/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
new file mode 100644
index 0000000..947722c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.rrd;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.rrd4j.core.RrdByteArrayBackend;
+
+/**
+ *
+ */
+public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable {
+
+ private final SolrRrdBackendFactory factory;
+ private final boolean readOnly;
+ private volatile boolean dirty = false;
+ private volatile boolean closed = false;
+
+ public SolrRrdBackend(String path, boolean readOnly, SolrRrdBackendFactory factory) {
+ super(path);
+ this.readOnly = readOnly;
+ this.factory = factory;
+ }
+
+ /**
+ * Open an unregistered read-only clone of the backend.
+ * @param other other backend
+ */
+ public SolrRrdBackend(SolrRrdBackend other) {
+ super(other.getPath());
+ readOnly = true;
+ factory = null;
+ byte[] otherBuffer = other.buffer;
+ buffer = new byte[otherBuffer.length];
+ System.arraycopy(otherBuffer, 0, buffer, 0, otherBuffer.length);
+ }
+
+ public boolean isReadOnly() {
+ return readOnly;
+ }
+
+ @Override
+ protected synchronized void write(long offset, byte[] bytes) throws IOException {
+ if (readOnly || closed) {
+ return;
+ }
+ super.write(offset, bytes);
+ dirty = true;
+ }
+
+ public byte[] maybeSync() {
+ if (readOnly || closed) {
+ return null;
+ }
+ if (!dirty) {
+ return null;
+ }
+ byte[] bufferCopy = new byte[buffer.length];
+ System.arraycopy(buffer, 0, bufferCopy, 0, buffer.length);
+ return bufferCopy;
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ closed = true;
+ if (factory != null) {
+ // unregister myself from the factory
+ factory.unregisterBackend(getPath());
+ }
+ // close
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71f03617/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
new file mode 100644
index 0000000..3e735b7
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.rrd;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrCloseable;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.IOUtils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.rrd4j.core.RrdBackend;
+import org.rrd4j.core.RrdBackendFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrCloseable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final int DEFAULT_SYNC_PERIOD = 60;
+ public static final int DEFAULT_MAX_DBS = 500;
+
+ public static final String ID_PREFIX = "rrd_";
+ public static final String DOC_TYPE = "rrd";
+
+ public static final String DATA_FIELD = "data_bin";
+
+ private final CoreContainer coreContainer;
+ private final String collection;
+ private ScheduledThreadPoolExecutor syncService;
+ private int syncPeriod = DEFAULT_SYNC_PERIOD;
+ private volatile boolean closed = false;
+ private boolean logMissingSystemColl = true;
+
+ private final Map<String, SolrRrdBackend> backends = new ConcurrentHashMap<>();
+
+ public SolrRrdBackendFactory(CoreContainer coreContainer, String collection) {
+ this.coreContainer = coreContainer;
+ this.collection = collection;
+ syncService = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1,
+ new DefaultSolrThreadFactory("SolrRrdBackendFactory-syncService"));
+ syncService.setRemoveOnCancelPolicy(true);
+ syncService.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
+ syncService.scheduleWithFixedDelay(() -> maybeSyncBackends(), syncPeriod, syncPeriod, TimeUnit.SECONDS);
+ }
+
+ private void ensureOpen() throws IOException {
+ if (closed) {
+ throw new IOException("Factory already closed");
+ }
+ }
+
+ @Override
+ protected synchronized RrdBackend open(String path, boolean readOnly) throws IOException {
+ ensureOpen();
+ SolrRrdBackend backend = backends.computeIfAbsent(path, p -> new SolrRrdBackend(p, readOnly, this));
+ if (backend.isReadOnly()) {
+ if (readOnly) {
+ return backend;
+ } else {
+ // replace it with a writable one
+ backend = new SolrRrdBackend(path, readOnly, this);
+ backends.put(path, backend);
+ return backend;
+ }
+ } else {
+ if (readOnly) {
+ // return a throwaway read-only copy
+ return new SolrRrdBackend(backend);
+ } else {
+ return backend;
+ }
+ }
+ }
+
+ CloudSolrClient getSolrClient() throws SolrException {
+ if (this.coreContainer.isZooKeeperAware()) {
+ return new CloudSolrClient.Builder(
+ Collections.singletonList(coreContainer.getZkController().getZkServerAddress()),
+ Optional.empty())
+ .build();
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "SolrRrd is not supported in non-cloud mode");
+ }
+ }
+
+ byte[] getData(String path) throws IOException {
+ try (CloudSolrClient client = getSolrClient()) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(CommonParams.Q, "{!term f=id}" + ID_PREFIX + path);
+ params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);
+ QueryResponse rsp = client.query(CollectionAdminParams.SYSTEM_COLL, params);
+ SolrDocumentList docs = rsp.getResults();
+ if (docs == null || docs.isEmpty()) {
+ return null;
+ }
+ if (docs.size() > 1) {
+ throw new SolrServerException("Expected at most 1 doc with id '" + ID_PREFIX + path + "' but got " + docs);
+ }
+ SolrDocument doc = docs.get(0);
+ Object o = doc.getFieldValue(DATA_FIELD);
+ if (o == null) {
+ return null;
+ }
+ if (o instanceof byte[]) {
+ return (byte[])o;
+ } else {
+ throw new SolrServerException("Unexpected value of '" + DATA_FIELD + "' field: " + o.getClass().getName() + ": " + o);
+ }
+ } catch (SolrServerException e) {
+ throw new IOException(e);
+ }
+ }
+
+ void unregisterBackend(String path) {
+ backends.remove(path);
+ }
+
+ public List<String> list() throws IOException {
+ ArrayList<String> names = new ArrayList<>();
+ try (CloudSolrClient client = getSolrClient()) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(CommonParams.Q, "*:*");
+ params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);
+ params.add(CommonParams.FL, "id");
+ params.add(CommonParams.SORT, "id asc");
+ params.add(CommonParams.ROWS, String.valueOf(DEFAULT_MAX_DBS));
+ QueryResponse rsp = client.query(CollectionAdminParams.SYSTEM_COLL, params);
+ SolrDocumentList docs = rsp.getResults();
+ if (docs != null) {
+ docs.forEach(d -> names.add(((String)d.getFieldValue("id")).substring(ID_PREFIX.length())));
+ }
+ } catch (SolrServerException e) {
+ log.warn("Error retrieving RRD list", e);
+ }
+ // add all doc id-s
+ return names;
+ }
+
+ public void remove(String path) throws IOException {
+ SolrRrdBackend backend = backends.get(path);
+ if (backend != null) {
+ IOUtils.closeQuietly(backend);
+ }
+ // remove Solr doc
+ try (CloudSolrClient client = getSolrClient()) {
+ client.deleteByQuery(CollectionAdminParams.SYSTEM_COLL, "{!term f=id}" + ID_PREFIX + path);
+ } catch (SolrServerException e) {
+ log.warn("Error deleting RRD for path " + path, e);
+ }
+ }
+
+ public synchronized void maybeSyncBackends() {
+ if (closed) {
+ return;
+ }
+ Map<String, byte[]> syncData = new HashMap<>();
+ backends.forEach((path, backend) -> {
+ byte[] data = backend.maybeSync();
+ if (data != null) {
+ syncData.put(backend.getPath(), data);
+ }
+ });
+ if (syncData.isEmpty()) {
+ return;
+ }
+ // write updates
+ try (CloudSolrClient client = getSolrClient()) {
+ ClusterState clusterState = client.getClusterStateProvider().getClusterState();
+ if (clusterState.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL) == null) {
+ if (logMissingSystemColl) {
+ log.warn("Collection " + CollectionAdminParams.SYSTEM_COLL + " missing, not persisting updates");
+ logMissingSystemColl = false;
+ }
+ }
+ logMissingSystemColl = true;
+ syncData.forEach((path, data) -> {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.setField("id", ID_PREFIX + path);
+ doc.addField(CommonParams.TYPE, DOC_TYPE);
+ doc.addField(DATA_FIELD, data);
+ try {
+ client.add(CollectionAdminParams.SYSTEM_COLL, doc);
+ } catch (SolrServerException | IOException e) {
+ log.warn("Error updating RRD data for " + path, e);
+ }
+ });
+ try {
+ client.commit(CollectionAdminParams.SYSTEM_COLL);
+ } catch (SolrServerException e) {
+ log.warn("Error committing RRD data updates", e);
+ }
+ } catch (IOException e) {
+ log.warn("Error sending RRD data updates", e);
+ }
+ }
+
+ @Override
+ protected boolean exists(String path) throws IOException {
+ try (CloudSolrClient client = getSolrClient()) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(CommonParams.Q, "{!term f=id}" + ID_PREFIX + path);
+ params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);
+ params.add(CommonParams.FL, "id");
+ QueryResponse rsp = client.query(CollectionAdminParams.SYSTEM_COLL, params);
+ SolrDocumentList docs = rsp.getResults();
+ if (docs == null || docs.isEmpty()) {
+ return false;
+ }
+ if (docs.size() > 1) {
+ throw new SolrServerException("Expected at most 1 doc with id '" + ID_PREFIX + path + "' but got " + docs);
+ }
+ return true;
+ } catch (SolrServerException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ protected boolean shouldValidateHeader(String path) throws IOException {
+ return false;
+ }
+
+ @Override
+ public String getName() {
+ return "SOLR";
+ }
+
+ @Override
+ public boolean isClosed() {
+ return closed;
+ }
+
+ @Override
+ public void close() throws IOException {
+ closed = true;
+ backends.forEach((p, b) -> IOUtils.closeQuietly(b));
+ backends.clear();
+ syncService.shutdown();
+ syncService = null;
+ }
+}
[3/3] lucene-solr:jira/solr-11779: SOLR-11779: Use SolrMetricManager
for managing metrics history. Add more functionality to
MetricsHistoryHandler.
Posted by ab...@apache.org.
SOLR-11779: Use SolrMetricManager for managing metrics history. Add more functionality
to MetricsHistoryHandler.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d778367b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d778367b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d778367b
Branch: refs/heads/jira/solr-11779
Commit: d778367b6346a32b93fd0c16569cc56d4c34e941
Parents: ac6aa8a
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Mon May 7 21:00:24 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Mon May 7 21:00:24 2018 +0200
----------------------------------------------------------------------
.../org/apache/solr/core/CoreContainer.java | 4 +
.../solr/handler/admin/MetricsHandler.java | 52 +++--
.../handler/admin/MetricsHistoryHandler.java | 191 ++++++++++++++++++-
.../apache/solr/metrics/SolrMetricManager.java | 41 +++-
.../apache/solr/metrics/rrd/SolrRrdBackend.java | 44 ++++-
.../solr/metrics/rrd/SolrRrdBackendFactory.java | 63 ++----
.../solr/security/PermissionNameProvider.java | 1 +
.../src/resources/apispec/metrics.history.json | 24 +++
8 files changed, 343 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d778367b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 74b718c..9caadd2 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -474,6 +474,10 @@ public class CoreContainer {
return metricManager;
}
+ public MetricsHandler getMetricsHandler() {
+ return metricsHandler;
+ }
+
//-------------------------------------------------------------------
// Initialization / Cleanup
//-------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d778367b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index ed1e474..3c8a152 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.BiConsumer;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -36,6 +37,7 @@ import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
@@ -89,17 +91,21 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Core container instance not initialized");
}
- boolean compact = req.getParams().getBool(COMPACT_PARAM, true);
- String[] keys = req.getParams().getParams(KEY_PARAM);
+ handleRequest(req.getParams(), (k, v) -> rsp.add(k, v));
+ }
+
+ public void handleRequest(SolrParams params, BiConsumer<String, Object> consumer) throws Exception {
+ boolean compact = params.getBool(COMPACT_PARAM, true);
+ String[] keys = params.getParams(KEY_PARAM);
if (keys != null && keys.length > 0) {
- handleKeyRequest(keys, req, rsp);
+ handleKeyRequest(keys, consumer);
return;
}
- MetricFilter mustMatchFilter = parseMustMatchFilter(req);
- MetricUtils.PropertyFilter propertyFilter = parsePropertyFilter(req);
- List<MetricType> metricTypes = parseMetricTypes(req);
+ MetricFilter mustMatchFilter = parseMustMatchFilter(params);
+ MetricUtils.PropertyFilter propertyFilter = parsePropertyFilter(params);
+ List<MetricType> metricTypes = parseMetricTypes(params);
List<MetricFilter> metricFilters = metricTypes.stream().map(MetricType::asMetricFilter).collect(Collectors.toList());
- Set<String> requestedRegistries = parseRegistries(req);
+ Set<String> requestedRegistries = parseRegistries(params);
NamedList response = new SimpleOrderedMap();
for (String registryName : requestedRegistries) {
@@ -111,10 +117,10 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
response.add(registryName, result);
}
}
- rsp.getValues().add("metrics", response);
+ consumer.accept("metrics", response);
}
- private void handleKeyRequest(String[] keys, SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ public void handleKeyRequest(String[] keys, BiConsumer<String, Object> consumer) throws Exception {
SimpleOrderedMap result = new SimpleOrderedMap();
SimpleOrderedMap errors = new SimpleOrderedMap();
for (String key : keys) {
@@ -153,9 +159,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
});
}
- rsp.getValues().add("metrics", result);
+ consumer.accept("metrics", result);
if (errors.size() > 0) {
- rsp.getValues().add("errors", errors);
+ consumer.accept("errors", errors);
}
}
@@ -174,8 +180,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return sb.toString();
}
- private MetricFilter parseMustMatchFilter(SolrQueryRequest req) {
- String[] prefixes = req.getParams().getParams(PREFIX_PARAM);
+ private MetricFilter parseMustMatchFilter(SolrParams params) {
+ String[] prefixes = params.getParams(PREFIX_PARAM);
MetricFilter prefixFilter = null;
if (prefixes != null && prefixes.length > 0) {
Set<String> prefixSet = new HashSet<>();
@@ -184,7 +190,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
prefixFilter = new SolrMetricManager.PrefixFilter(prefixSet);
}
- String[] regexes = req.getParams().getParams(REGEX_PARAM);
+ String[] regexes = params.getParams(REGEX_PARAM);
MetricFilter regexFilter = null;
if (regexes != null && regexes.length > 0) {
regexFilter = new SolrMetricManager.RegexFilter(regexes);
@@ -204,8 +210,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return mustMatchFilter;
}
- private MetricUtils.PropertyFilter parsePropertyFilter(SolrQueryRequest req) {
- String[] props = req.getParams().getParams(PROPERTY_PARAM);
+ private MetricUtils.PropertyFilter parsePropertyFilter(SolrParams params) {
+ String[] props = params.getParams(PROPERTY_PARAM);
if (props == null || props.length == 0) {
return MetricUtils.PropertyFilter.ALL;
}
@@ -222,9 +228,13 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
}
- private Set<String> parseRegistries(SolrQueryRequest req) {
- String[] groupStr = req.getParams().getParams(GROUP_PARAM);
- String[] registryStr = req.getParams().getParams(REGISTRY_PARAM);
+ private Set<String> parseRegistries(SolrParams params) {
+ String[] groupStr = params.getParams(GROUP_PARAM);
+ String[] registryStr = params.getParams(REGISTRY_PARAM);
+ return parseRegistries(groupStr, registryStr);
+ }
+
+ public Set<String> parseRegistries(String[] groupStr, String[] registryStr) {
if ((groupStr == null || groupStr.length == 0) && (registryStr == null || registryStr.length == 0)) {
// return all registries
return container.getMetricManager().registryNames();
@@ -278,8 +288,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return validRegistries;
}
- private List<MetricType> parseMetricTypes(SolrQueryRequest req) {
- String[] typeStr = req.getParams().getParams(TYPE_PARAM);
+ private List<MetricType> parseMetricTypes(SolrParams params) {
+ String[] typeStr = params.getParams(TYPE_PARAM);
List<String> types = Collections.emptyList();
if (typeStr != null && typeStr.length > 0) {
types = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d778367b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
index cc372cf..5db09dd 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
@@ -16,14 +16,185 @@
*/
package org.apache.solr.handler.admin;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.solr.api.Api;
+import org.apache.solr.api.ApiBag;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.TimeSource;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.metrics.rrd.SolrRrdBackendFactory;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.security.AuthorizationContext;
+import org.apache.solr.security.PermissionNameProvider;
+import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.rrd4j.ConsolFun;
+import org.rrd4j.DsType;
+import org.rrd4j.core.RrdBackendFactory;
+import org.rrd4j.core.RrdDb;
+import org.rrd4j.core.RrdDef;
+import org.rrd4j.core.Sample;
/**
*
*/
-public class MetricsHistoryHandler extends RequestHandlerBase {
+public class MetricsHistoryHandler extends RequestHandlerBase implements PermissionNameProvider {
+
+ public static final Set<String> DEFAULT_CORE_COUNTERS = new HashSet<String>() {{
+ add("QUERY./select.requests");
+ add("INDEX.sizeInBytes");
+ add("UPDATE./update.requests");
+ }};
+ public static final Set<String> DEFAULT_CORE_GAUGES = new HashSet<String>() {{
+ add("INDEX.sizeInBytes");
+ }};
+ public static final Set<String> DEFAULT_NODE_GAUGES = new HashSet<String>() {{
+ add("CONTAINER.fs.coreRoot.usableSpace");
+ }};
+ public static final Set<String> DEFAULT_JVM_GAUGES = new HashSet<String>() {{
+ add("memory.heap.used");
+ add("os.processCpuLoad");
+ add("os.systemLoadAverage");
+ }};
+
+ public static final int DEFAULT_COLLECT_PERIOD = 60;
+ public static final String URI_PREFIX = "solr:///";
+
+ private final SolrRrdBackendFactory factory;
+ private final MetricsHandler metricsHandler;
+ private final SolrMetricManager metricManager;
+ private final ScheduledThreadPoolExecutor collectService;
+ private final TimeSource timeSource;
+ private final int collectPeriod;
+ private final Map<String, Set<String>> counters = new HashMap<>();
+ private final Map<String, Set<String>> gauges = new HashMap<>();
+
+ public MetricsHistoryHandler(CoreContainer coreContainer) {
+ factory = new SolrRrdBackendFactory(new CloudSolrClient.Builder(
+ Collections.singletonList(coreContainer.getZkController().getZkServerAddress()),
+ Optional.empty())
+ .withHttpClient(coreContainer.getUpdateShardHandler().getHttpClient())
+ .build(), CollectionAdminParams.SYSTEM_COLL);
+ RrdBackendFactory.registerAndSetAsDefaultFactory(factory);
+ metricsHandler = coreContainer.getMetricsHandler();
+ metricManager = coreContainer.getMetricManager();
+ collectPeriod = DEFAULT_COLLECT_PERIOD;
+ timeSource = coreContainer.getZkController().getSolrCloudManager().getTimeSource();
+
+ counters.put(Group.core.toString(), DEFAULT_CORE_COUNTERS);
+ counters.put(Group.node.toString(), Collections.emptySet());
+ counters.put(Group.jvm.toString(), Collections.emptySet());
+ gauges.put(Group.core.toString(), DEFAULT_CORE_GAUGES);
+ gauges.put(Group.node.toString(), DEFAULT_NODE_GAUGES);
+ gauges.put(Group.jvm.toString(), DEFAULT_JVM_GAUGES);
+
+ collectService = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(2,
+ new DefaultSolrThreadFactory("SolrRrdBackendFactory"));
+ collectService.setRemoveOnCancelPolicy(true);
+ collectService.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
+ collectService.scheduleWithFixedDelay(() -> collectMetrics(), collectPeriod, collectPeriod, TimeUnit.SECONDS);
+ }
+
+ private void collectMetrics() {
+ // get metrics
+ for (Group group : Arrays.asList(Group.core, Group.node, Group.jvm)) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(MetricsHandler.GROUP_PARAM, group.toString());
+ params.add(MetricsHandler.COMPACT_PARAM, "true");
+ counters.get(group.toString()).forEach(c -> params.add(MetricsHandler.PREFIX_PARAM, c));
+ gauges.get(group.toString()).forEach(c -> params.add(MetricsHandler.PREFIX_PARAM, c));
+ AtomicReference<Object> result = new AtomicReference<>();
+ try {
+ metricsHandler.handleRequest(params, (k, v) -> {
+ if (k.equals("metrics")) {
+ result.set(v);
+ }
+ });
+ NamedList nl = (NamedList)result.get();
+ if (nl != null) {
+ for (Iterator<Map.Entry<String, Object>> it = nl.iterator(); it.hasNext(); ) {
+ Map.Entry<String, Object> entry = it.next();
+ final String registry = entry.getKey();
+ RrdDb db = metricManager.getOrCreateMetricHistory(registry, () -> {
+ RrdDef def = createDef(registry, group);
+ try {
+ RrdDb newDb = new RrdDb(def);
+ return newDb;
+ } catch (IOException e) {
+ return null;
+ }
+ });
+ if (db == null) {
+ continue;
+ }
+ Sample s = db.createSample(TimeUnit.MILLISECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS));
+ NamedList<Object> values = (NamedList<Object>)entry.getValue();
+ AtomicBoolean dirty = new AtomicBoolean(false);
+ counters.get(group.toString()).forEach(c -> {
+ Number val = (Number)values.get(c);
+ if (val != null) {
+ dirty.set(true);
+ s.setValue(c, val.doubleValue());
+ }
+ });
+ gauges.get(group.toString()).forEach(c -> {
+ Number val = (Number)values.get(c);
+ if (val != null) {
+ dirty.set(true);
+ s.setValue(c, val.doubleValue());
+ }
+ });
+ if (dirty.get()) {
+ s.update();
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ RrdDef createDef(String registry, Group group) {
+ registry = SolrMetricManager.overridableRegistryName(registry);
+
+ RrdDef def = new RrdDef(URI_PREFIX + registry, collectPeriod);
+ def.setStartTime(TimeUnit.MILLISECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS));
+
+ // add datasources
+ counters.get(group.toString()).forEach(c ->
+ def.addDatasource(c, DsType.COUNTER, collectPeriod * 2, Double.NaN, Double.NaN));
+ gauges.get(group.toString()).forEach(g ->
+ def.addDatasource(g, DsType.GAUGE, collectPeriod * 2, Double.NaN, Double.NaN));
+
+ // add archives
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 120); // 2 hours
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 10, 288); // 48 hours
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 60, 336); // 2 weeks
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 240, 180); // 2 months
+ return def;
+ }
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
@@ -32,6 +203,22 @@ public class MetricsHistoryHandler extends RequestHandlerBase {
@Override
public String getDescription() {
- return null;
+ return "A handler for metrics history";
+ }
+
+ @Override
+ public Name getPermissionName(AuthorizationContext request) {
+ return Name.METRICS_HISTORY_READ_PERM;
+ }
+
+ @Override
+ public Boolean registerV2() {
+ return Boolean.TRUE;
+ }
+
+ @Override
+ public Collection<Api> getApis() {
+ return ApiBag.wrapRequestHandlers(this, "metrics.history");
}
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d778367b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
index d5b8864..5c3b49a 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
@@ -34,6 +34,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;
@@ -48,6 +49,7 @@ import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.MetricSet;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
+import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.MetricsConfig;
@@ -55,6 +57,7 @@ import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.core.SolrResourceLoader;
+import org.rrd4j.core.RrdDb;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -97,6 +100,8 @@ public class SolrMetricManager {
private final Map<String, Map<String, SolrMetricReporter>> reporters = new HashMap<>();
+ private final ConcurrentHashMap<String, RrdDb> metricHistories = new ConcurrentHashMap<>();
+
private final Lock reportersLock = new ReentrantLock();
private final Lock swapLock = new ReentrantLock();
@@ -423,14 +428,14 @@ public class SolrMetricManager {
} else {
swapLock.lock();
try {
- return getOrCreate(registries, registry);
+ return getOrCreateRegistry(registries, registry);
} finally {
swapLock.unlock();
}
}
}
- private static MetricRegistry getOrCreate(ConcurrentMap<String, MetricRegistry> map, String registry) {
+ private static MetricRegistry getOrCreateRegistry(ConcurrentMap<String, MetricRegistry> map, String registry) {
final MetricRegistry existing = map.get(registry);
if (existing == null) {
final MetricRegistry created = new MetricRegistry();
@@ -445,6 +450,26 @@ public class SolrMetricManager {
}
}
+ public RrdDb getOrCreateMetricHistory(String registry, Supplier<RrdDb> supplier) {
+ registry = overridableRegistryName(registry);
+ final RrdDb existing = metricHistories.get(registry);
+ if (existing == null) {
+ final RrdDb created = supplier.get();
+ if (created == null) {
+ // maybe someone else succeeded
+ return metricHistories.get(registry);
+ }
+ final RrdDb raced = metricHistories.putIfAbsent(registry, created);
+ if (raced == null) {
+ return created;
+ } else {
+ return raced;
+ }
+ } else {
+ return existing;
+ }
+ }
+
/**
* Remove a named registry.
* @param registry name of the registry to remove
@@ -464,6 +489,10 @@ public class SolrMetricManager {
swapLock.unlock();
}
}
+ RrdDb db = metricHistories.remove(registry);
+ if (db != null) {
+ IOUtils.closeQuietly(db);
+ }
}
/**
@@ -490,12 +519,20 @@ public class SolrMetricManager {
}
MetricRegistry reg1 = registries.remove(registry1);
MetricRegistry reg2 = registries.remove(registry2);
+ RrdDb db1 = metricHistories.remove(registry1);
+ RrdDb db2 = metricHistories.remove(registry2);
if (reg2 != null) {
registries.put(registry1, reg2);
}
if (reg1 != null) {
registries.put(registry2, reg1);
}
+ if (db2 != null) {
+ metricHistories.put(registry1, db2);
+ }
+ if (db1 != null) {
+ metricHistories.put(registry2, db1);
+ }
} finally {
swapLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d778367b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
index 947722c..c395a53 100644
--- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
@@ -18,27 +18,42 @@ package org.apache.solr.metrics.rrd;
import java.io.Closeable;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.concurrent.locks.ReentrantLock;
import org.rrd4j.core.RrdByteArrayBackend;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
*/
public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final SolrRrdBackendFactory factory;
private final boolean readOnly;
+ private final ReentrantLock lock = new ReentrantLock();
private volatile boolean dirty = false;
private volatile boolean closed = false;
public SolrRrdBackend(String path, boolean readOnly, SolrRrdBackendFactory factory) {
super(path);
- this.readOnly = readOnly;
this.factory = factory;
+ try {
+ byte[] data = factory.getData(path);
+ if (data != null) {
+ this.buffer = data;
+ }
+ } catch (IOException e) {
+ log.warn("Exception retrieving data from " + path + ", store will be readOnly", e);
+ readOnly = true;
+ }
+ this.readOnly = readOnly;
}
/**
- * Open an unregistered read-only clone of the backend.
+ * Open an unregistered (throwaway) read-only clone of another backend.
* @param other other backend
*/
public SolrRrdBackend(SolrRrdBackend other) {
@@ -55,24 +70,35 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable {
}
@Override
- protected synchronized void write(long offset, byte[] bytes) throws IOException {
+ protected void write(long offset, byte[] bytes) throws IOException {
if (readOnly || closed) {
return;
}
- super.write(offset, bytes);
- dirty = true;
+ lock.lock();
+ try {
+ super.write(offset, bytes);
+ dirty = true;
+ } finally {
+ lock.unlock();
+ }
}
- public byte[] maybeSync() {
+ public byte[] getSyncData() {
if (readOnly || closed) {
return null;
}
if (!dirty) {
return null;
}
- byte[] bufferCopy = new byte[buffer.length];
- System.arraycopy(buffer, 0, bufferCopy, 0, buffer.length);
- return bufferCopy;
+ // hold a lock to block writes so that we get consistent data
+ lock.lock();
+ try {
+ byte[] bufferCopy = new byte[buffer.length];
+ System.arraycopy(buffer, 0, bufferCopy, 0, buffer.length);
+ return bufferCopy;
+ } finally {
+ lock.unlock();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d778367b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
index 3e735b7..1401ee3 100644
--- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
@@ -19,20 +19,18 @@ package org.apache.solr.metrics.rrd;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.io.SolrClientCache;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrCloseable;
import org.apache.solr.common.SolrDocument;
@@ -61,24 +59,23 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
public static final int DEFAULT_MAX_DBS = 500;
public static final String ID_PREFIX = "rrd_";
- public static final String DOC_TYPE = "rrd";
+ public static final String DOC_TYPE = "metrics_rrd";
public static final String DATA_FIELD = "data_bin";
- private final CoreContainer coreContainer;
+ private final SolrClient solrClient;
private final String collection;
private ScheduledThreadPoolExecutor syncService;
private int syncPeriod = DEFAULT_SYNC_PERIOD;
private volatile boolean closed = false;
- private boolean logMissingSystemColl = true;
private final Map<String, SolrRrdBackend> backends = new ConcurrentHashMap<>();
- public SolrRrdBackendFactory(CoreContainer coreContainer, String collection) {
- this.coreContainer = coreContainer;
+ public SolrRrdBackendFactory(SolrClient solrClient, String collection) {
+ this.solrClient = solrClient;
this.collection = collection;
- syncService = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1,
- new DefaultSolrThreadFactory("SolrRrdBackendFactory-syncService"));
+ syncService = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(2,
+ new DefaultSolrThreadFactory("SolrRrdBackendFactory"));
syncService.setRemoveOnCancelPolicy(true);
syncService.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
syncService.scheduleWithFixedDelay(() -> maybeSyncBackends(), syncPeriod, syncPeriod, TimeUnit.SECONDS);
@@ -113,23 +110,12 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
}
}
- CloudSolrClient getSolrClient() throws SolrException {
- if (this.coreContainer.isZooKeeperAware()) {
- return new CloudSolrClient.Builder(
- Collections.singletonList(coreContainer.getZkController().getZkServerAddress()),
- Optional.empty())
- .build();
- } else {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "SolrRrd is not supported in non-cloud mode");
- }
- }
-
byte[] getData(String path) throws IOException {
- try (CloudSolrClient client = getSolrClient()) {
+ try {
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CommonParams.Q, "{!term f=id}" + ID_PREFIX + path);
params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);
- QueryResponse rsp = client.query(CollectionAdminParams.SYSTEM_COLL, params);
+ QueryResponse rsp = solrClient.query(collection, params);
SolrDocumentList docs = rsp.getResults();
if (docs == null || docs.isEmpty()) {
return null;
@@ -158,14 +144,14 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
public List<String> list() throws IOException {
ArrayList<String> names = new ArrayList<>();
- try (CloudSolrClient client = getSolrClient()) {
+ try {
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CommonParams.Q, "*:*");
params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);
params.add(CommonParams.FL, "id");
params.add(CommonParams.SORT, "id asc");
params.add(CommonParams.ROWS, String.valueOf(DEFAULT_MAX_DBS));
- QueryResponse rsp = client.query(CollectionAdminParams.SYSTEM_COLL, params);
+ QueryResponse rsp = solrClient.query(collection, params);
SolrDocumentList docs = rsp.getResults();
if (docs != null) {
docs.forEach(d -> names.add(((String)d.getFieldValue("id")).substring(ID_PREFIX.length())));
@@ -173,7 +159,6 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
} catch (SolrServerException e) {
log.warn("Error retrieving RRD list", e);
}
- // add all doc id-s
return names;
}
@@ -183,8 +168,8 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
IOUtils.closeQuietly(backend);
}
// remove Solr doc
- try (CloudSolrClient client = getSolrClient()) {
- client.deleteByQuery(CollectionAdminParams.SYSTEM_COLL, "{!term f=id}" + ID_PREFIX + path);
+ try {
+ solrClient.deleteByQuery(collection, "{!term f=id}" + ID_PREFIX + path);
} catch (SolrServerException e) {
log.warn("Error deleting RRD for path " + path, e);
}
@@ -196,7 +181,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
}
Map<String, byte[]> syncData = new HashMap<>();
backends.forEach((path, backend) -> {
- byte[] data = backend.maybeSync();
+ byte[] data = backend.getSyncData();
if (data != null) {
syncData.put(backend.getPath(), data);
}
@@ -205,28 +190,20 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
return;
}
// write updates
- try (CloudSolrClient client = getSolrClient()) {
- ClusterState clusterState = client.getClusterStateProvider().getClusterState();
- if (clusterState.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL) == null) {
- if (logMissingSystemColl) {
- log.warn("Collection " + CollectionAdminParams.SYSTEM_COLL + " missing, not persisting updates");
- logMissingSystemColl = false;
- }
- }
- logMissingSystemColl = true;
+ try {
syncData.forEach((path, data) -> {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("id", ID_PREFIX + path);
doc.addField(CommonParams.TYPE, DOC_TYPE);
doc.addField(DATA_FIELD, data);
try {
- client.add(CollectionAdminParams.SYSTEM_COLL, doc);
+ solrClient.add(collection, doc);
} catch (SolrServerException | IOException e) {
log.warn("Error updating RRD data for " + path, e);
}
});
try {
- client.commit(CollectionAdminParams.SYSTEM_COLL);
+ solrClient.commit(collection);
} catch (SolrServerException e) {
log.warn("Error committing RRD data updates", e);
}
@@ -237,12 +214,12 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
@Override
protected boolean exists(String path) throws IOException {
- try (CloudSolrClient client = getSolrClient()) {
+ try {
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CommonParams.Q, "{!term f=id}" + ID_PREFIX + path);
params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);
params.add(CommonParams.FL, "id");
- QueryResponse rsp = client.query(CollectionAdminParams.SYSTEM_COLL, params);
+ QueryResponse rsp = solrClient.query(collection, params);
SolrDocumentList docs = rsp.getResults();
if (docs == null || docs.isEmpty()) {
return false;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d778367b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
index 4073947..79b4d29 100644
--- a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
+++ b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
@@ -50,6 +50,7 @@ public interface PermissionNameProvider {
AUTOSCALING_READ_PERM("autoscaling-read", null),
AUTOSCALING_WRITE_PERM("autoscaling-write", null),
AUTOSCALING_HISTORY_READ_PERM("autoscaling-history-read", null),
+ METRICS_HISTORY_READ_PERM("metrics-history-read", null),
ALL("all", unmodifiableSet(new HashSet<>(asList("*", null))))
;
final String name;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d778367b/solr/solrj/src/resources/apispec/metrics.history.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/resources/apispec/metrics.history.json b/solr/solrj/src/resources/apispec/metrics.history.json
new file mode 100644
index 0000000..49de7b9
--- /dev/null
+++ b/solr/solrj/src/resources/apispec/metrics.history.json
@@ -0,0 +1,24 @@
+{
+ "documentation": "https://lucene.apache.org/solr/guide/metrics-reporting.html",
+ "description": "Metrics history handler allows retrieving samples of past metrics recorded in the .system collection.",
+ "methods": [
+ "GET"
+ ],
+ "url": {
+ "paths": [
+ "/cluster/metrics/history"
+ ],
+ "params": {
+ "collection": {
+ "type": "string",
+ "description": "Collection where metrics history is stored, '.system' by default.",
+ "default": ".system"
+ },
+ "q": {
+ "type": "string",
+ "description": "Arbitrary query to limit the selected metrics.",
+ "default": "*:*"
+ }
+ }
+ }
+}
[2/3] lucene-solr:jira/solr-11779: Merge branch 'master' into
jira/solr-11779
Posted by ab...@apache.org.
Merge branch 'master' into jira/solr-11779
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ac6aa8a1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ac6aa8a1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ac6aa8a1
Branch: refs/heads/jira/solr-11779
Commit: ac6aa8a1de4c47eacdf9b12e5d02351ff42f709f
Parents: 71f0361 d702dc6
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue Apr 24 12:33:10 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue Apr 24 12:33:10 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 9 +-
.../lucene/index/BufferedUpdatesStream.java | 134 +---
.../apache/lucene/index/DocumentsWriter.java | 224 +++---
.../index/DocumentsWriterFlushControl.java | 4 +-
.../lucene/index/DocumentsWriterPerThread.java | 32 +-
.../lucene/index/FrozenBufferedUpdates.java | 76 +-
.../org/apache/lucene/index/IndexWriter.java | 646 ++++++-----------
.../org/apache/lucene/index/ReaderPool.java | 390 ++++++++++
.../apache/lucene/index/ReadersAndUpdates.java | 6 +-
.../lucene/index/StandardDirectoryReader.java | 4 +-
.../search/DisjunctionMatchesIterator.java | 10 +-
.../apache/lucene/search/MatchesIterator.java | 8 -
.../lucene/search/TermMatchesIterator.java | 9 +-
.../org/apache/lucene/search/TermQuery.java | 2 +-
.../lucene/index/TestIndexWriterDelete.java | 24 +
.../lucene/index/TestIndexWriterExceptions.java | 9 +-
.../org/apache/lucene/index/TestInfoStream.java | 8 +-
.../lucene/index/TestPendingSoftDeletes.java | 5 +-
.../org/apache/lucene/index/TestReaderPool.java | 223 ++++++
.../lucene/search/TestMatchesIterator.java | 73 +-
lucene/ivy-versions.properties | 2 +-
.../search/TestInetAddressRangeQueries.java | 2 +
.../spatial3d/geom/GeoComplexPolygon.java | 347 +++++----
.../spatial3d/geom/GeoPolygonFactory.java | 20 +-
.../lucene/spatial3d/geom/GeoPolygonTest.java | 81 ++-
.../apache/lucene/index/RandomIndexWriter.java | 8 +-
.../lucene/search/AssertingMatchesIterator.java | 7 -
solr/CHANGES.txt | 48 +-
solr/bin/solr | 4 +
solr/bin/solr.cmd | 3 +
solr/bin/solr.in.cmd | 12 +-
solr/bin/solr.in.sh | 16 +-
...ractNamedEntitiesUpdateProcessorFactory.java | 6 +
.../carrot2/CarrotClusteringEngine.java | 2 +-
...anguageIdentifierUpdateProcessorFactory.java | 8 +-
...OpenNLPLangDetectUpdateProcessorFactory.java | 8 +-
...anguageIdentifierUpdateProcessorFactory.java | 8 +-
.../solrj/embedded/EmbeddedSolrServer.java | 24 +-
.../cloud/api/collections/SetAliasPropCmd.java | 45 +-
.../cloud/autoscaling/IndexSizeTrigger.java | 4 +-
.../cloud/autoscaling/SearchRateTrigger.java | 487 +++++++++++--
.../apache/solr/core/HdfsDirectoryFactory.java | 13 +-
.../apache/solr/core/MMapDirectoryFactory.java | 2 +-
.../solr/core/NRTCachingDirectoryFactory.java | 2 +-
.../src/java/org/apache/solr/core/SolrCore.java | 24 +-
.../apache/solr/handler/CdcrRequestHandler.java | 8 +-
.../apache/solr/handler/RequestHandlerBase.java | 4 +-
.../solr/handler/UpdateRequestHandler.java | 2 +-
.../solr/handler/admin/CollectionsHandler.java | 142 ++--
.../solr/handler/admin/ConfigSetsHandler.java | 18 +-
.../handler/admin/MetricsCollectorHandler.java | 2 +-
.../component/QueryElevationComponent.java | 2 +-
.../solr/highlight/HighlightingPluginBase.java | 2 +-
.../solr/request/LocalSolrQueryRequest.java | 10 +-
.../solr/response/XSLTResponseWriter.java | 2 +-
.../org/apache/solr/schema/IndexSchema.java | 2 +-
.../solr/schema/ManagedIndexSchemaFactory.java | 2 +-
.../solr/spelling/DirectSolrSpellChecker.java | 2 +-
.../org/apache/solr/update/SolrIndexConfig.java | 4 +
.../org/apache/solr/update/TransactionLog.java | 3 +-
.../ClassificationUpdateProcessorFactory.java | 2 +-
...oreCommitOptimizeUpdateProcessorFactory.java | 2 +-
.../processor/LogUpdateProcessorFactory.java | 2 +-
.../processor/RegexpBoostProcessorFactory.java | 2 +-
.../SignatureUpdateProcessorFactory.java | 2 +-
.../processor/URLClassifyProcessorFactory.java | 2 +-
.../solrconfig-concurrentmergescheduler.xml | 37 +
.../org/apache/solr/BasicFunctionalityTest.java | 2 +-
.../apache/solr/cloud/AliasIntegrationTest.java | 33 +-
.../org/apache/solr/cloud/CloudTestUtils.java | 8 +-
.../solr/cloud/CreateRoutedAliasTest.java | 10 +-
.../solr/cloud/TestMiniSolrCloudClusterSSL.java | 59 ++
.../autoscaling/AutoScalingHandlerTest.java | 4 +-
.../cloud/autoscaling/NodeLostTriggerTest.java | 1 +
.../SearchRateTriggerIntegrationTest.java | 592 +++++++++++++--
.../autoscaling/SearchRateTriggerTest.java | 201 +++++-
.../cloud/autoscaling/sim/TestLargeCluster.java | 5 +-
.../autoscaling/sim/TestTriggerIntegration.java | 12 +-
.../solr/handler/TestReplicationHandler.java | 36 +-
.../solr/handler/admin/TestCollectionAPIs.java | 23 +
.../request/TestUnInvertedFieldException.java | 8 +-
.../apache/solr/update/SolrIndexConfigTest.java | 24 +
.../apache/solr/update/TransactionLogTest.java | 47 ++
.../TimeRoutedAliasUpdateProcessorTest.java | 31 +-
solr/licenses/commons-fileupload-1.3.2.jar.sha1 | 1 -
solr/licenses/commons-fileupload-1.3.3.jar.sha1 | 1 +
solr/solr-ref-guide/src/about-this-guide.adoc | 2 +
solr/solr-ref-guide/src/blob-store-api.adoc | 96 ++-
solr/solr-ref-guide/src/config-api.adoc | 720 ++++++++++++++-----
solr/solr-ref-guide/src/config-sets.adoc | 36 +-
solr/solr-ref-guide/src/configsets-api.adoc | 244 ++++---
.../src/configuring-solrconfig-xml.adoc | 42 +-
solr/solr-ref-guide/src/css/customstyles.css | 2 +-
solr/solr-ref-guide/src/enabling-ssl.adoc | 21 +-
.../src/implicit-requesthandlers.adoc | 374 ++++++++--
solr/solr-ref-guide/src/learning-to-rank.adoc | 2 +
.../src/requestdispatcher-in-solrconfig.adoc | 2 +-
solr/solr-ref-guide/src/schema-api.adoc | 2 +-
...tting-up-an-external-zookeeper-ensemble.adoc | 403 ++++++++---
.../src/solrcloud-autoscaling-triggers.adoc | 106 ++-
.../src/update-request-processors.adoc | 2 +-
.../client/solrj/cloud/autoscaling/Clause.java | 21 +-
.../autoscaling/DelegatingCloudManager.java | 2 +-
.../cloud/autoscaling/DeleteNodeSuggester.java | 46 ++
.../autoscaling/DeleteReplicaSuggester.java | 74 ++
.../client/solrj/cloud/autoscaling/Operand.java | 2 +-
.../client/solrj/cloud/autoscaling/Policy.java | 38 +-
.../solrj/cloud/autoscaling/ReplicaCount.java | 6 +
.../solrj/cloud/autoscaling/ReplicaInfo.java | 18 +
.../cloud/autoscaling/SplitShardSuggester.java | 6 +
.../solrj/cloud/autoscaling/Suggester.java | 8 +-
.../solrj/cloud/autoscaling/Suggestion.java | 4 +-
.../solrj/cloud/autoscaling/Violation.java | 2 +-
.../solr/client/solrj/impl/HttpClientUtil.java | 59 +-
.../solrj/impl/SolrClientCloudManager.java | 2 +-
.../solrj/impl/SolrClientNodeStateProvider.java | 4 +-
.../org/apache/solr/client/solrj/io/Lang.java | 1 +
.../client/solrj/io/eval/MemsetEvaluator.java | 167 +++++
.../solrj/io/graph/GatherNodesStream.java | 20 +-
.../client/solrj/io/stream/FacetStream.java | 6 +-
.../solr/client/solrj/io/stream/LetStream.java | 11 +-
.../solr/client/solrj/io/stream/SqlStream.java | 3 +-
.../solrj/io/stream/TimeSeriesStream.java | 4 +-
.../solrj/request/CollectionAdminRequest.java | 11 +
.../request/JavaBinUpdateRequestCodec.java | 6 +-
.../java/org/apache/solr/common/MapWriter.java | 9 +-
.../org/apache/solr/common/cloud/Replica.java | 2 +
.../apache/solr/common/cloud/ZkStateReader.java | 22 +-
.../apache/solr/common/params/SolrParams.java | 22 +-
.../org/apache/solr/common/util/NamedList.java | 30 +
.../apispec/cluster.configs.Commands.json | 12 +-
.../apispec/cluster.configs.delete.json | 2 +-
.../src/resources/apispec/cluster.configs.json | 2 +-
.../solrj/cloud/autoscaling/TestPolicy.java | 22 +-
.../client/solrj/impl/HttpClientUtilTest.java | 108 +++
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 106 +++
.../solr/common/params/SolrParamTest.java | 38 +-
.../org/apache/solr/util/SSLTestConfig.java | 89 ++-
...estConfig.hostname-and-ip-missmatch.keystore | Bin 0 -> 2246 bytes
.../resources/SSLTestConfig.testing.keystore | Bin 2208 -> 2207 bytes
.../src/resources/create-keystores.sh | 37 +
solr/webapp/web/css/angular/collections.css | 4 -
solr/webapp/web/css/angular/cores.css | 8 -
solr/webapp/web/partials/cores.html | 2 -
145 files changed, 5538 insertions(+), 1985 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ac6aa8a1/lucene/ivy-versions.properties
----------------------------------------------------------------------