You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/12/18 11:31:48 UTC
lucene-solr:jira/solr-12259: Fix remaining issues in the porting.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-12259 010ca9e54 -> cf6054394
Fix remaining issues in the porting.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cf605439
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cf605439
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cf605439
Branch: refs/heads/jira/solr-12259
Commit: cf605439412e642235cf440b3b01b1ffed504d83
Parents: 010ca9e
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue Dec 18 12:31:19 2018 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue Dec 18 12:31:19 2018 +0100
----------------------------------------------------------------------
.../apache/solr/core/SolrResourceLoader.java | 4 +-
.../apache/solr/handler/admin/ColStatus.java | 190 ++++++++++++++++
.../solr/handler/admin/CollectionsHandler.java | 13 ++
.../admin/SegmentsInfoRequestHandler.java | 216 +++++++++++++++++--
.../org/apache/solr/update/SolrIndexWriter.java | 20 +-
.../solr/collection1/conf/schema-docValues.xml | 6 +-
.../solr/cloud/BasicDistributedZkTest.java | 2 +-
.../solr/index/ConcurrentIndexUpgradeTest.java | 40 ++--
.../solr/index/IndexUpgradeIntegrationTest.java | 8 +-
.../index/WrapperMergePolicyFactoryTest.java | 2 +-
.../solrj/request/CollectionAdminRequest.java | 44 ++++
.../solr/common/params/CollectionParams.java | 3 +-
12 files changed, 496 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 0ff5c7b..b9eafd5 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -66,6 +66,7 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.handler.component.ShardHandlerFactory;
+import org.apache.solr.index.MergePolicyFactory;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.rest.RestManager;
@@ -834,7 +835,8 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
ShardHandlerFactory.class,
SimilarityFactory.class,
SolrRequestHandler.class,
- UpdateRequestProcessorFactory.class
+ UpdateRequestProcessorFactory.class,
+ MergePolicyFactory.class
}
);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/core/src/java/org/apache/solr/handler/admin/ColStatus.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ColStatus.java b/solr/core/src/java/org/apache/solr/handler/admin/ColStatus.java
new file mode 100644
index 0000000..3976e86
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ColStatus.java
@@ -0,0 +1,190 @@
+package org.apache.solr.handler.admin;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.http.client.HttpClient;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.io.SolrClientCache;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.RoutingRule;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkCoreNodeProps;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class ColStatus {
+ private static final Logger log = LoggerFactory.getLogger(ColStatus.class);
+
+ private final ZkStateReader zkStateReader;
+ private final ZkNodeProps props;
+ private final SolrClientCache solrClientCache;
+
+ public static final String FIELD_INFOS_PROP = "fieldInfos";
+ public static final String SEGMENTS_PROP = "segments";
+ public static final String DV_STATS_PROP = "dvStats";
+
+ public ColStatus(HttpClient httpClient, ZkStateReader zkStateReader, ZkNodeProps props) {
+ this.props = props;
+ this.solrClientCache = new SolrClientCache(httpClient);
+ this.zkStateReader = zkStateReader;
+ }
+
+ public void getColStatus(NamedList<Object> results)
+ throws KeeperException, InterruptedException {
+ ClusterState clusterState = zkStateReader.getClusterState();
+ Collection<String> collections;
+ String col = props.getStr(ZkStateReader.COLLECTION_PROP);
+ if (col == null) {
+ collections = new HashSet<>(clusterState.getCollectionsMap().keySet());
+ } else {
+ collections = Collections.singleton(col);
+ }
+ boolean withFieldInfos = props.getBool(FIELD_INFOS_PROP, false);
+ boolean withSegments = props.getBool(SEGMENTS_PROP, false);
+ boolean withDVStats = props.getBool(DV_STATS_PROP, false);
+ for (String collection : collections) {
+ DocCollection coll = clusterState.getCollectionOrNull(collection);
+ if (coll == null) {
+ continue;
+ }
+ SimpleOrderedMap<Object> colMap = new SimpleOrderedMap<>();
+ if (coll.getReplicationFactor() != null) {
+ colMap.add("replicationFactor", coll.getReplicationFactor());
+ }
+ colMap.add("stateFormat", coll.getStateFormat());
+ colMap.add("znodeVersion", coll.getZNodeVersion());
+ colMap.add("autoAddReplicas", coll.getAutoAddReplicas());
+ colMap.add("maxShardsPerNode", coll.getMaxShardsPerNode());
+ colMap.add("activeSlices", coll.getActiveSlices().size());
+ colMap.add("inactiveSlices", coll.getSlices().size() - coll.getActiveSlices().size());
+ results.add(collection, colMap);
+
+ Set<String> nonCompliant = new TreeSet<>();
+
+ SimpleOrderedMap<Object> slices = new SimpleOrderedMap<>();
+ for (Slice s : coll.getSlices()) {
+ SimpleOrderedMap<Object> sliceMap = new SimpleOrderedMap<>();
+ slices.add(s.getName(), sliceMap);
+ SimpleOrderedMap<Object> replicaMap = new SimpleOrderedMap<>();
+ int totalReplicas = s.getReplicas().size();
+ int activeReplicas = 0;
+ int downReplicas = 0;
+ int recoveringReplicas = 0;
+ int recoveryFailedReplicas = 0;
+ for (Replica r : s.getReplicas()) {
+ switch (r.getState()) {
+ case ACTIVE:
+ activeReplicas++;
+ break;
+ case DOWN:
+ downReplicas++;
+ break;
+ case RECOVERING:
+ recoveringReplicas++;
+ break;
+ case RECOVERY_FAILED:
+ recoveryFailedReplicas++;
+ break;
+ }
+ }
+ replicaMap.add("total", totalReplicas);
+ replicaMap.add("active", activeReplicas);
+ replicaMap.add("down", downReplicas);
+ replicaMap.add("recovering", recoveringReplicas);
+ replicaMap.add("recovery_failed", recoveryFailedReplicas);
+ sliceMap.add("state", s.getState().toString());
+ sliceMap.add("range", s.getRange().toString());
+ Map<String, RoutingRule> rules = s.getRoutingRules();
+ if (rules != null && !rules.isEmpty()) {
+ sliceMap.add("routingRules", rules);
+ }
+ sliceMap.add("replicas", replicaMap);
+ Replica leader = s.getLeader();
+ if (leader == null) { // pick the first one
+ leader = s.getReplicas().size() > 0 ? s.getReplicas().iterator().next() : null;
+ }
+ if (leader == null) {
+ continue;
+ }
+ SimpleOrderedMap<Object> leaderMap = new SimpleOrderedMap<>();
+ sliceMap.add("leader", leaderMap);
+ leaderMap.add("coreNode", leader.getName());
+ leaderMap.addAll(leader.getProperties());
+ String url = ZkCoreNodeProps.getCoreUrl(leader);
+ try (SolrClient client = solrClientCache.getHttpSolrClient(url)) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(CommonParams.QT, "/admin/segments");
+ params.add("fieldInfos", "true");
+ params.add(DV_STATS_PROP, String.valueOf(withDVStats));
+ QueryRequest req = new QueryRequest(params);
+ NamedList<Object> rsp = client.request(req);
+ rsp.remove("responseHeader");
+ leaderMap.add("segInfos", rsp);
+ NamedList<Object> segs = (NamedList<Object>)rsp.get("segments");
+ if (segs != null) {
+ for (Map.Entry<String, Object> entry : segs) {
+ NamedList<Object> fields = (NamedList<Object>)((NamedList<Object>)entry.getValue()).get("fields");
+ if (fields != null) {
+ for (Map.Entry<String, Object> fEntry : fields) {
+ Object nc = ((NamedList<Object>)fEntry.getValue()).get("nonCompliant");
+ if (nc != null) {
+ nonCompliant.add(fEntry.getKey());
+ }
+ }
+ }
+ if (!withFieldInfos) {
+ ((NamedList<Object>)entry.getValue()).remove("fields");
+ }
+ }
+ }
+ if (!withSegments) {
+ rsp.remove("segments");
+ }
+ } catch (SolrServerException | IOException e) {
+ log.warn("Error getting details of replica segments from " + url, e);
+ }
+ }
+ if (nonCompliant.isEmpty()) {
+ nonCompliant.add("(NONE)");
+ }
+ colMap.add("schemaNonCompliant", nonCompliant);
+ colMap.add("slices", slices);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index c593be6..6d6a2fd 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -853,6 +853,19 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
rsp.getValues().addAll(response.getResponse());
return null;
}),
+ COLSTATUS_OP(COLSTATUS, (req, rsp, h) -> {
+ Map<String, Object> all = copy(req.getParams(), null,
+ CoreAdminParams.NAME, COLLECTION_PROP,
+ ColStatus.FIELD_INFOS_PROP, ColStatus.SEGMENTS_PROP, ColStatus.DV_STATS_PROP);
+ // make sure we can get the name if there's "name" but not "collection"
+ if (all.containsKey(CoreAdminParams.NAME) && !all.containsKey(COLLECTION_PROP)) {
+ all.put(COLLECTION_PROP, all.get(CoreAdminParams.NAME));
+ }
+ new ColStatus(h.coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
+ h.coreContainer.getZkController().getZkStateReader(), new ZkNodeProps(all))
+ .getColStatus(rsp.getValues());
+ return null;
+ }),
/**
* Handle cluster status request.
* Can return status per specific collection/shard or per all collections.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/core/src/java/org/apache/solr/handler/admin/SegmentsInfoRequestHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SegmentsInfoRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SegmentsInfoRequestHandler.java
index 740280b..c10b807 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/SegmentsInfoRequestHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/SegmentsInfoRequestHandler.java
@@ -17,68 +17,123 @@
package org.apache.solr.handler.admin;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergePolicy.MergeSpecification;
import org.apache.lucene.index.MergePolicy.OneMerge;
import org.apache.lucene.index.MergeTrigger;
import org.apache.lucene.index.SegmentCommitInfo;
import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.util.Version;
+import org.apache.solr.common.luke.FieldFlag;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.uninverting.UninvertingReader;
+import org.apache.solr.update.SolrIndexWriter;
import org.apache.solr.util.RefCounted;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static org.apache.lucene.index.IndexOptions.DOCS;
+import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS;
+import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
import static org.apache.solr.common.params.CommonParams.NAME;
/**
* This handler exposes information about last commit generation segments
*/
public class SegmentsInfoRequestHandler extends RequestHandlerBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
- rsp.add("segments", getSegmentsInfo(req, rsp));
+ getSegmentsInfo(req, rsp);
rsp.setHttpCaching(false);
}
- private SimpleOrderedMap<Object> getSegmentsInfo(SolrQueryRequest req, SolrQueryResponse rsp)
+ private void getSegmentsInfo(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
SolrIndexSearcher searcher = req.getSearcher();
+ IndexSchema schema = req.getSchema();
+ SolrCore core = req.getCore();
+ RefCounted<IndexWriter> iwRef = core.getSolrCoreState().getIndexWriter(core);
+ SimpleOrderedMap<Object> infosInfo = new SimpleOrderedMap<>();
+ SimpleOrderedMap<Object> coreInfo = new SimpleOrderedMap<>();
+ infosInfo.add("core", coreInfo);
+ coreInfo.add("startTime", core.getStartTimeStamp());
+ coreInfo.add("reader", searcher.getIndexReader().toString());
+
+ if (iwRef != null) {
+ try {
+ IndexWriter iw = iwRef.get();
+ MergePolicy mp = iw.getConfig().getMergePolicy();
+ coreInfo.add("mergePolicy", mp.getClass().getName());
+ } finally {
+ iwRef.decref();
+ }
+ }
SegmentInfos infos =
SegmentInfos.readLatestCommit(searcher.getIndexReader().directory());
+ Version minVersion = infos.getMinSegmentLuceneVersion();
+ if (minVersion != null) {
+ infosInfo.add("minSegmentLuceneVersion", minVersion.toString());
+ }
+ Version commitVersion = infos.getCommitLuceneVersion();
+ if (commitVersion != null) {
+ infosInfo.add("commitLuceneVersion", commitVersion.toString());
+ }
+ infosInfo.add("numSegments", infos.size());
+ infosInfo.add("segmentsFileName", infos.getSegmentsFileName());
+ infosInfo.add("userData", infos.userData);
- List<String> mergeCandidates = getMergeCandidatesNames(req, infos);
+ List<String> mergeCandidates = new ArrayList<>();
+ SimpleOrderedMap<Object> runningMerges = getMergeInformation(req, infos, mergeCandidates);
SimpleOrderedMap<Object> segmentInfos = new SimpleOrderedMap<>();
SimpleOrderedMap<Object> segmentInfo = null;
- List<SegmentCommitInfo> sortable = new ArrayList<>();
- sortable.addAll(infos.asList());
- // Order by the number of live docs. The display is logarithmic so it is a little jumbled visually
- sortable.sort((s1, s2) -> {
- return (s2.info.maxDoc() - s2.getDelCount()) - (s1.info.maxDoc() - s1.getDelCount());
- });
- for (SegmentCommitInfo segmentCommitInfo : sortable) {
- segmentInfo = getSegmentInfo(segmentCommitInfo);
+ boolean withFieldInfos = req.getParams().getBool("fieldInfos", false);
+ boolean withDVStats = req.getParams().getBool("dvStats", false);
+
+ List<LeafReaderContext> leafContexts = searcher.getIndexReader().leaves();
+ for (SegmentCommitInfo segmentCommitInfo : infos) {
+ segmentInfo = getSegmentInfo(segmentCommitInfo, withFieldInfos, withDVStats, leafContexts, schema);
if (mergeCandidates.contains(segmentCommitInfo.info.name)) {
segmentInfo.add("mergeCandidate", true);
}
segmentInfos.add((String) segmentInfo.get(NAME), segmentInfo);
}
- return segmentInfos;
+ rsp.add("info", infosInfo);
+ if (runningMerges.size() > 0) {
+ rsp.add("runningMerges", runningMerges);
+ }
+ rsp.add("segments", segmentInfos);
}
private SimpleOrderedMap<Object> getSegmentInfo(
- SegmentCommitInfo segmentCommitInfo) throws IOException {
+ SegmentCommitInfo segmentCommitInfo, boolean withFieldInfos, boolean withDVStats,
+ List<LeafReaderContext> leafContexts, IndexSchema schema) throws IOException {
SimpleOrderedMap<Object> segmentInfoMap = new SimpleOrderedMap<>();
segmentInfoMap.add(NAME, segmentCommitInfo.info.name);
@@ -90,16 +145,145 @@ public class SegmentsInfoRequestHandler extends RequestHandlerBase {
segmentInfoMap.add("age", new Date(timestamp));
segmentInfoMap.add("source",
segmentCommitInfo.info.getDiagnostics().get("source"));
+ if (!segmentCommitInfo.info.getDiagnostics().isEmpty()) {
+ segmentInfoMap.add("diagnostics", segmentCommitInfo.info.getDiagnostics());
+ }
+ if (!segmentCommitInfo.info.getAttributes().isEmpty()) {
+ segmentInfoMap.add("attributes", segmentCommitInfo.info.getAttributes());
+ }
segmentInfoMap.add("version", segmentCommitInfo.info.getVersion().toString());
+ if (withFieldInfos) {
+ SegmentReader seg = null;
+ for (LeafReaderContext lrc : leafContexts) {
+ LeafReader leafReader = lrc.reader();
+ // unwrap
+ while (leafReader instanceof FilterLeafReader) {
+ leafReader = ((FilterLeafReader)leafReader).getDelegate();
+ }
+ if (leafReader instanceof SegmentReader) {
+ SegmentReader sr = (SegmentReader)leafReader;
+ if (sr.getSegmentInfo().info.equals(segmentCommitInfo.info)) {
+ seg = sr;
+ break;
+ }
+ }
+ }
+ if (seg == null) {
+ log.debug("Skipping segment info - not available as a SegmentReader: " + segmentCommitInfo);
+ } else {
+ FieldInfos fis = seg.getFieldInfos();
+ SimpleOrderedMap<Object> fields = new SimpleOrderedMap<>();
+ for (FieldInfo fi : fis) {
+ fields.add(fi.name, getFieldFlags(seg, fi, withDVStats, schema));
+ }
+ segmentInfoMap.add("fields", fields);
+ }
+ }
return segmentInfoMap;
}
- private List<String> getMergeCandidatesNames(SolrQueryRequest req, SegmentInfos infos) throws IOException {
- List<String> result = new ArrayList<String>();
+ private SimpleOrderedMap<Object> getFieldFlags(SegmentReader reader, FieldInfo fi, boolean withDVStats, IndexSchema schema) {
+
+ SimpleOrderedMap<Object> fieldFlags = new SimpleOrderedMap<>();
+ StringBuilder flags = new StringBuilder();
+
+ IndexOptions opts = fi.getIndexOptions();
+ flags.append( (opts != IndexOptions.NONE) ? FieldFlag.INDEXED.getAbbreviation() : '-' );
+ DocValuesType dvt = fi.getDocValuesType();
+ if (dvt != DocValuesType.NONE) {
+ flags.append(FieldFlag.DOC_VALUES.getAbbreviation());
+ switch (dvt) {
+ case NUMERIC:
+ flags.append("num");
+ break;
+ case BINARY:
+ flags.append("bin");
+ break;
+ case SORTED:
+ flags.append("srt");
+ break;
+ case SORTED_NUMERIC:
+ flags.append("srn");
+ break;
+ case SORTED_SET:
+ flags.append("srs");
+ break;
+ }
+ if (withDVStats) {
+ try {
+ fieldFlags.add("dvStats", UninvertingReader.getDVStats(reader, fi));
+ } catch (IOException e) {
+ fieldFlags.add("dvStats", "ERROR: " + e.toString());
+ }
+ }
+ } else {
+ flags.append("----");
+ }
+ flags.append( (fi.hasVectors()) ? FieldFlag.TERM_VECTOR_STORED.getAbbreviation() : '-' );
+ flags.append( (fi.omitsNorms()) ? FieldFlag.OMIT_NORMS.getAbbreviation() : '-' );
+
+ flags.append( (DOCS == opts ) ?
+ FieldFlag.OMIT_TF.getAbbreviation() : '-' );
+
+ flags.append((DOCS_AND_FREQS == opts) ?
+ FieldFlag.OMIT_POSITIONS.getAbbreviation() : '-');
+
+ flags.append((DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS == opts) ?
+ FieldFlag.STORE_OFFSETS_WITH_POSITIONS.getAbbreviation() : '-');
+
+ fieldFlags.add("flags", flags.toString());
+
+ // probably too much detail?
+// Map<String, String> attributes = fi.attributes();
+// if (!attributes.isEmpty()) {
+// fieldFlags.add("attributes", attributes);
+// }
+ // check compliance with the current schema
+ SchemaField sf = schema.getFieldOrNull(fi.name);
+
+ if (sf != null) {
+ SimpleOrderedMap<Object> nonCompliant = new SimpleOrderedMap<>();
+ if (sf.hasDocValues() &&
+ fi.getDocValuesType() == DocValuesType.NONE &&
+ fi.getIndexOptions() != IndexOptions.NONE) {
+ nonCompliant.add("docValues", "schema=" + sf.getType().getUninversionType(sf) + ", segment=false");
+ }
+ if (!sf.hasDocValues() &&
+ fi.getDocValuesType() != DocValuesType.NONE &&
+ fi.getIndexOptions() != IndexOptions.NONE) {
+ nonCompliant.add("docValues", "schema=false, segment=" + fi.getDocValuesType().toString());
+ }
+ if (!sf.isPolyField()) { // difficult to find all sub-fields in a general way
+ if (sf.indexed() != (fi.getIndexOptions() != IndexOptions.NONE)) {
+ nonCompliant.add("indexed", "schema=" + sf.indexed() + ", segment=" + fi.getIndexOptions());
+ }
+ }
+ if (sf.omitNorms() != fi.omitsNorms()) {
+ nonCompliant.add("omitNorms", "schema=" + sf.omitNorms() + ", segment=" + fi.omitsNorms());
+ }
+ if (sf.storeTermVector() != fi.hasVectors()) {
+ nonCompliant.add("termVectors", "schema=" + sf.storeTermVector() + ", segment=" + fi.hasVectors());
+ }
+ if (sf.storeOffsetsWithPositions() != (fi.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)) {
+ nonCompliant.add("storeOffsetsWithPositions", "schema=" + sf.storeOffsetsWithPositions() + ", segment=" + fi.getIndexOptions());
+ }
+
+ if (nonCompliant.size() > 0) {
+ fieldFlags.add("nonCompliant", nonCompliant);
+ }
+ }
+ return fieldFlags;
+ }
+
+ private SimpleOrderedMap<Object> getMergeInformation(SolrQueryRequest req, SegmentInfos infos, List<String> mergeCandidates) throws IOException {
+ SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
RefCounted<IndexWriter> refCounted = req.getCore().getSolrCoreState().getIndexWriter(req.getCore());
try {
IndexWriter indexWriter = refCounted.get();
+ if (indexWriter instanceof SolrIndexWriter) {
+ result.addAll(((SolrIndexWriter)indexWriter).getRunningMerges());
+ }
//get chosen merge policy
MergePolicy mp = indexWriter.getConfig().getMergePolicy();
//Find merges
@@ -108,7 +292,7 @@ public class SegmentsInfoRequestHandler extends RequestHandlerBase {
for (OneMerge merge : findMerges.merges) {
//TODO: add merge grouping
for (SegmentCommitInfo mergeSegmentInfo : merge.segments) {
- result.add(mergeSegmentInfo.info.name);
+ mergeCandidates.add(mergeSegmentInfo.info.name);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
index fda5fa5..5d1d02d 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
@@ -18,8 +18,10 @@ package org.apache.solr.update;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -85,6 +87,8 @@ public class SolrIndexWriter extends IndexWriter {
private final AtomicInteger runningMinorMergesSegments = new AtomicInteger();
private final AtomicLong runningMajorMergesDocs = new AtomicLong();
private final AtomicLong runningMinorMergesDocs = new AtomicLong();
+ // merge diagnostics
+ private final Map<String, Long> runningMerges = new ConcurrentHashMap<>();
private final SolrMetricManager metricManager;
private final String registryName;
@@ -192,12 +196,18 @@ public class SolrIndexWriter extends IndexWriter {
// we override this method to collect metrics for merges.
@Override
public void merge(MergePolicy.OneMerge merge) throws IOException {
+ String segString = merge.segString();
+ long totalNumDocs = merge.totalNumDocs();
if (!mergeTotals) {
- super.merge(merge);
+ try {
+ runningMerges.put(segString, totalNumDocs);
+ super.merge(merge);
+ } finally {
+ runningMerges.remove(segString);
+ }
return;
}
long deletedDocs = 0;
- long totalNumDocs = merge.totalNumDocs();
for (SegmentCommitInfo info : merge.segments) {
totalNumDocs -= info.getDelCount();
deletedDocs += info.getDelCount();
@@ -221,12 +231,14 @@ public class SolrIndexWriter extends IndexWriter {
context = minorMerge.time();
}
try {
+ runningMerges.put(segString, totalNumDocs);
super.merge(merge);
} catch (Throwable t) {
mergeErrors.inc();
throw t;
} finally {
context.stop();
+ runningMerges.remove(segString);
if (major) {
runningMajorMerges.decrementAndGet();
runningMajorMergesDocs.addAndGet(-totalNumDocs);
@@ -239,6 +251,10 @@ public class SolrIndexWriter extends IndexWriter {
}
}
+ public Map<String, Object> getRunningMerges() {
+ return Collections.unmodifiableMap(runningMerges);
+ }
+
@Override
protected void doAfterFlush() throws IOException {
if (flushMeter != null) { // this is null when writer is used only for snapshot cleanup
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml
index 4a5ab17..4288bf7 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml
@@ -53,10 +53,8 @@
<fieldType name="uuid" class="solr.UUIDField"/>
- <field name="id" type="string" required="true"/>
- <!-- int varient of id, adding this here simplifies some indexing to do numeric sorting -->
- <field name="id_i" type="int" indexed="true" stored="true" multiValued="false" />
- <copyField source="id" dest="id_i" />
+ <field name="id" type="string" indexed="true" stored="true" required="true"/>
+ <field name="_version_" type="long" indexed="true" stored="true"/>
<field name="floatdv" type="float" indexed="false" stored="false" docValues="true" default="1"/>
<field name="intdv" type="int" indexed="false" stored="false" docValues="true" default="2"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
index 895fa29..c95ae85 100644
--- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
@@ -239,7 +239,7 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"});
String[] vals = new String[100];
for (int i=0; i<100; i++) {
- vals[i] = "code/test " + i;
+ vals[i] = "test " + i;
}
indexr(id, 17, "SubjectTerms_mfacet", vals);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/core/src/test/org/apache/solr/index/ConcurrentIndexUpgradeTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/index/ConcurrentIndexUpgradeTest.java b/solr/core/src/test/org/apache/solr/index/ConcurrentIndexUpgradeTest.java
index 6c6ca30..4e4937c 100644
--- a/solr/core/src/test/org/apache/solr/index/ConcurrentIndexUpgradeTest.java
+++ b/solr/core/src/test/org/apache/solr/index/ConcurrentIndexUpgradeTest.java
@@ -28,6 +28,7 @@ import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -133,8 +134,7 @@ public class ConcurrentIndexUpgradeTest extends AbstractFullDistribZkTestBase {
// make sure we've indexed some documents
Thread.sleep(5000);
- CollectionAdminRequest<CollectionAdminRequest.ColStatus> status = new CollectionAdminRequest.ColStatus()
- .setCollectionName(collectionName)
+ CollectionAdminRequest.ColStatus status = CollectionAdminRequest.colStatus(collectionName)
.setWithFieldInfos(true)
.setWithSegments(true);
CollectionAdminResponse rsp = status.process(cloudClient);
@@ -149,9 +149,8 @@ public class ConcurrentIndexUpgradeTest extends AbstractFullDistribZkTestBase {
// prevent merging
pluginProps.put(AddDocValuesMergePolicyFactory.NO_MERGE_PROP, true);
String propValue = Utils.toJSONString(pluginProps);
- CollectionAdminRequest.ClusterProp clusterProp = new CollectionAdminRequest.ClusterProp()
- .setPropertyName(PluggableMergePolicyFactory.MERGE_POLICY_PROP + collectionName)
- .setPropertyValue(propValue);
+ CollectionAdminRequest.ClusterProp clusterProp = CollectionAdminRequest
+ .setClusterProperty(PluggableMergePolicyFactory.MERGE_POLICY_PROP + collectionName, propValue);
clusterProp.process(cloudClient);
log.info("-- completed set cluster props");
@@ -174,8 +173,7 @@ public class ConcurrentIndexUpgradeTest extends AbstractFullDistribZkTestBase {
// bounce the collection
Map<String, Long> urlToTimeBefore = new HashMap<>();
collectStartTimes(collectionName, cloudClient, urlToTimeBefore);
- CollectionAdminRequest<CollectionAdminRequest.Reload> reload = new CollectionAdminRequest.Reload()
- .setCollectionName(collectionName);
+ CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(collectionName);
rsp = reload.process(cloudClient);
boolean reloaded = waitForReloads(collectionName, cloudClient, urlToTimeBefore);
@@ -195,9 +193,7 @@ public class ConcurrentIndexUpgradeTest extends AbstractFullDistribZkTestBase {
// update plugin props to allow merging
pluginProps.put(AddDocValuesMergePolicyFactory.NO_MERGE_PROP, false);
propValue = Utils.toJSONString(pluginProps);
- clusterProp = new CollectionAdminRequest.ClusterProp()
- .setPropertyName(PluggableMergePolicyFactory.MERGE_POLICY_PROP + collectionName)
- .setPropertyValue(propValue);
+ clusterProp = CollectionAdminRequest.setClusterProperty(PluggableMergePolicyFactory.MERGE_POLICY_PROP + collectionName, propValue);
clusterProp.process(cloudClient);
log.info("-- completed set cluster props 2");
@@ -232,21 +228,11 @@ public class ConcurrentIndexUpgradeTest extends AbstractFullDistribZkTestBase {
// verify that all docs have docValues
for (JettySolrRunner jetty : jettys) {
- CoreContainer cores = ((SolrDispatchFilter)jetty.getDispatchFilter().getFilter()).getCores();
+ CoreContainer cores = jetty.getCoreContainer();
for (SolrCore core : cores.getCores()) {
RefCounted<SolrIndexSearcher> searcherRef = core.getSearcher();
SolrIndexSearcher searcher = searcherRef.get();
try {
- LeafReader reader = searcher.getLeafReader();
- int maxDoc = reader.maxDoc();
- SortedDocValues dvs = reader.getSortedDocValues(TEST_FIELD);
- for (int i = 0; i < maxDoc; i++) {
- Document d = reader.document(i);
- BytesRef bytes = dvs.get(i);
- assertNotNull(bytes);
- String dvString = bytes.utf8ToString();
- assertEquals(d.get("id"), dvString);
- }
DirectoryReader directoryReader = searcher.getIndexReader();
for (LeafReaderContext leafCtx : directoryReader.leaves()) {
LeafReader leaf = leafCtx.reader();
@@ -261,6 +247,18 @@ public class ConcurrentIndexUpgradeTest extends AbstractFullDistribZkTestBase {
if (marker != null) {
assertEquals(AddDocValuesMergePolicyFactory.DEFAULT_MARKER, marker);
}
+ // use the wrapped reader here
+ SortedDocValues dvs = leaf.getSortedDocValues(TEST_FIELD);
+ int expected = leaf.numDocs();
+ int actual = 0;
+ while (dvs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+ Document d = leaf.document(dvs.docID());
+ BytesRef bytes = dvs.binaryValue();
+ assertNotNull(bytes);
+ assertTrue(bytes.toString(), bytes.length > 0);
+ String dvString = bytes.utf8ToString();
+ assertEquals(d.get("id"), dvString);
+ }
}
} finally {
searcherRef.decref();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/core/src/test/org/apache/solr/index/IndexUpgradeIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/index/IndexUpgradeIntegrationTest.java b/solr/core/src/test/org/apache/solr/index/IndexUpgradeIntegrationTest.java
index a50a8d2..9b403b2 100644
--- a/solr/core/src/test/org/apache/solr/index/IndexUpgradeIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/index/IndexUpgradeIntegrationTest.java
@@ -65,9 +65,8 @@ public class IndexUpgradeIntegrationTest extends AbstractFullDistribZkTestBase {
Map<String, Object> pluginProps = new HashMap<>();
pluginProps.put(FieldType.CLASS_NAME, AddDocValuesMergePolicyFactory.class.getName());
String propValue = Utils.toJSONString(pluginProps);
- CollectionAdminRequest.ClusterProp clusterProp = new CollectionAdminRequest.ClusterProp()
- .setPropertyName(PluggableMergePolicyFactory.MERGE_POLICY_PROP + collectionName)
- .setPropertyValue(propValue);
+ CollectionAdminRequest.ClusterProp clusterProp = CollectionAdminRequest
+ .setClusterProperty(PluggableMergePolicyFactory.MERGE_POLICY_PROP + collectionName, propValue);
clusterProp.process(cloudClient);
log.info("-- completed set cluster props");
@@ -115,8 +114,7 @@ public class IndexUpgradeIntegrationTest extends AbstractFullDistribZkTestBase {
log.info("-- completed collection reload");
// verify that schema doesn't match the actual fields anymore
- CollectionAdminRequest<CollectionAdminRequest.ColStatus> status = new CollectionAdminRequest.ColStatus()
- .setCollectionName(collectionName)
+ CollectionAdminRequest.ColStatus status = CollectionAdminRequest.colStatus(collectionName)
.setWithFieldInfos(true)
.setWithSegments(true);
CollectionAdminResponse rsp = status.process(cloudClient);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/core/src/test/org/apache/solr/index/WrapperMergePolicyFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/index/WrapperMergePolicyFactoryTest.java b/solr/core/src/test/org/apache/solr/index/WrapperMergePolicyFactoryTest.java
index 1d830ca..d416e13 100644
--- a/solr/core/src/test/org/apache/solr/index/WrapperMergePolicyFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/index/WrapperMergePolicyFactoryTest.java
@@ -52,7 +52,7 @@ public class WrapperMergePolicyFactoryTest extends SolrTestCaseJ4 {
final double testMaxMergedSegmentMB = defaultTMP.getMaxMergedSegmentMB() * 10;
final MergePolicyFactoryArgs args = new MergePolicyFactoryArgs();
- args.add(WrapperMergePolicyFactory.WRAPPED_PREFIX, "code/test");
+ args.add(WrapperMergePolicyFactory.WRAPPED_PREFIX, "test");
args.add("test.class", TieredMergePolicyFactory.class.getName());
args.add("test.maxMergeAtOnce", testMaxMergeAtOnce);
args.add("test.maxMergedSegmentMB", testMaxMergedSegmentMB);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 4f26984..0ccd618 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -1089,6 +1089,50 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
}
}
+ public static ColStatus colStatus(String collection) {
+ return new ColStatus(collection);
+ }
+
+ public static class ColStatus extends AsyncCollectionSpecificAdminRequest {
+ protected Boolean withSegments = null;
+ protected Boolean withFieldInfos = null;
+ protected Boolean withDVStats = null;
+
+ public ColStatus(String collection) {
+ super(CollectionAction.COLSTATUS, checkNotNull(CoreAdminParams.COLLECTION, collection));
+ }
+
+ public ColStatus setWithSegments(boolean withSegments) {
+ this.withSegments = withSegments;
+ return this;
+ }
+
+ public ColStatus setWithFieldInfos(boolean withFieldInfos) {
+ this.withFieldInfos = withFieldInfos;
+ return this;
+ }
+
+ public ColStatus setWithDVStats(boolean withDVStats) {
+ this.withDVStats = withDVStats;
+ return this;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams)super.getParams();
+ if (withSegments != null) {
+ params.add("segments", withSegments.toString());
+ }
+ if (withFieldInfos != null) {
+ params.add("fieldInfos", withFieldInfos.toString());
+ }
+ if (withDVStats != null) {
+ params.add("dvStats", withDVStats.toString());
+ }
+ return params;
+ }
+ }
+
/**
* Returns a SolrRequest to create a new shard in a collection
*/
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf605439/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index dee2f5f..7443dc3 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@ -121,7 +121,8 @@ public interface CollectionParams {
MOCK_REPLICA_TASK(false, LockLevel.REPLICA),
NONE(false, LockLevel.NONE),
// TODO: not implemented yet
- MERGESHARDS(true, LockLevel.SHARD)
+ MERGESHARDS(true, LockLevel.SHARD),
+ COLSTATUS(false, LockLevel.NONE)
;
public final boolean isWrite;