You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/10/23 00:05:31 UTC
[10/52] [abbrv] [partial] lucene-solr:jira/gradle: Add gradle support
for Solr
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperInfoHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperInfoHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperInfoHandler.java
deleted file mode 100644
index 0616ac8..0000000
--- a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperInfoHandler.java
+++ /dev/null
@@ -1,857 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.admin;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Reader;
-import java.io.Writer;
-import java.lang.invoke.MethodHandles;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.lucene.util.BytesRef;
-import org.apache.solr.cloud.ZkController;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.cloud.OnReconnect;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.MapSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.handler.RequestHandlerBase;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.JSONResponseWriter;
-import org.apache.solr.response.RawResponseWriter;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.util.SimplePostTool.BAOS;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.data.Stat;
-import org.apache.zookeeper.server.ByteBufferInputStream;
-import org.noggit.CharArr;
-import org.noggit.JSONWriter;
-import org.noggit.ObjectBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.common.params.CommonParams.OMIT_HEADER;
-import static org.apache.solr.common.params.CommonParams.PATH;
-import static org.apache.solr.common.params.CommonParams.WT;
-
-
-/**
- * Zookeeper Info
- *
- * @since solr 4.0
- */
-public final class ZookeeperInfoHandler extends RequestHandlerBase {
- private final CoreContainer cores;
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- // used for custom sorting collection names looking like prefix##
- // only go out to 7 digits (which safely fits in an int)
- private static final Pattern endsWithDigits = Pattern.compile("^(\\D*)(\\d{1,7}?)$");
-
- public ZookeeperInfoHandler(CoreContainer cc) {
- this.cores = cc;
- }
-
-
- @Override
- public String getDescription() {
- return "Fetch Zookeeper contents";
- }
-
- @Override
- public Category getCategory() {
- return Category.ADMIN;
- }
-
- /**
- * Enumeration of ways to filter collections on the graph panel.
- */
- static enum FilterType {
- none, name, status
- }
-
- /**
- * Holds state of a single page of collections requested from the cloud panel.
- */
- static final class PageOfCollections {
- List<String> selected;
- int numFound = 0; // total number of matches (across all pages)
- int start = 0;
- int rows = -1;
- FilterType filterType;
- String filter;
-
- PageOfCollections(int start, int rows, FilterType filterType, String filter) {
- this.start = start;
- this.rows = rows;
- this.filterType = filterType;
- this.filter = filter;
- }
-
- void selectPage(List<String> collections) {
- numFound = collections.size();
- // start with full set and then find the sublist for the desired selected
- selected = collections;
-
- if (rows > 0) { // paging desired
- if (start > numFound)
- start = 0; // this might happen if they applied a new filter
-
- int lastIndex = Math.min(start + rows, numFound);
- if (start > 0 || lastIndex < numFound)
- selected = collections.subList(start, lastIndex);
- }
- }
-
- /**
- * Filters a list of collections by name if applicable.
- */
- List<String> applyNameFilter(List<String> collections) {
-
- if (filterType != FilterType.name || filter == null)
- return collections; // name filter doesn't apply
-
- // typically, a user will type a prefix and then *, e.g. tj*
- // when they really mean tj.*
- String regexFilter = (!filter.endsWith(".*") && filter.endsWith("*"))
- ? filter.substring(0, filter.length() - 1) + ".*" : filter;
-
- // case-insensitive
- if (!regexFilter.startsWith("(?i)"))
- regexFilter = "(?i)" + regexFilter;
-
- Pattern filterRegex = Pattern.compile(regexFilter);
- List<String> filtered = new ArrayList<String>();
- for (String next : collections) {
- if (matches(filterRegex, next))
- filtered.add(next);
- }
-
- return filtered;
- }
-
- /**
- * Walk the collection state JSON object to see if it has any replicas that match
- * the state the user is filtering by.
- */
- @SuppressWarnings("unchecked")
- final boolean matchesStatusFilter(Map<String, Object> collectionState, Set<String> liveNodes) {
-
- if (filterType != FilterType.status || filter == null || filter.length() == 0)
- return true; // no status filter, so all match
-
- boolean isHealthy = true; // means all replicas for all shards active
- boolean hasDownedShard = false; // means one or more shards is down
- boolean replicaInRecovery = false;
-
- Map<String, Object> shards = (Map<String, Object>) collectionState.get("shards");
- for (String shardId : shards.keySet()) {
- boolean hasActive = false;
- Map<String, Object> shard = (Map<String, Object>) shards.get(shardId);
- Map<String, Object> replicas = (Map<String, Object>) shard.get("replicas");
- for (String replicaId : replicas.keySet()) {
- Map<String, Object> replicaState = (Map<String, Object>) replicas.get(replicaId);
- Replica.State coreState = Replica.State.getState((String) replicaState.get(ZkStateReader.STATE_PROP));
- String nodeName = (String) replicaState.get("node_name");
-
- // state can lie to you if the node is offline, so need to reconcile with live_nodes too
- if (!liveNodes.contains(nodeName))
- coreState = Replica.State.DOWN; // not on a live node, so must be down
-
- if (coreState == Replica.State.ACTIVE) {
- hasActive = true; // assumed no replicas active and found one that is for this shard
- } else {
- if (coreState == Replica.State.RECOVERING) {
- replicaInRecovery = true;
- }
- isHealthy = false; // assumed healthy and found one replica that is not
- }
- }
-
- if (!hasActive)
- hasDownedShard = true; // this is bad
- }
-
- if ("healthy".equals(filter)) {
- return isHealthy;
- } else if ("degraded".equals(filter)) {
- return !hasDownedShard && !isHealthy; // means no shards offline but not 100% healthy either
- } else if ("downed_shard".equals(filter)) {
- return hasDownedShard;
- } else if (Replica.State.getState(filter) == Replica.State.RECOVERING) {
- return !isHealthy && replicaInRecovery;
- }
-
- return true;
- }
-
- final boolean matches(final Pattern filter, final String collName) {
- return filter.matcher(collName).matches();
- }
-
- String getPagingHeader() {
- return start + "|" + rows + "|" + numFound + "|" + (filterType != null ? filterType.toString() : "") + "|" + (filter != null ? filter : "");
- }
-
- public String toString() {
- return getPagingHeader();
- }
-
- }
-
- /**
- * Supports paged navigation of collections on the cloud panel. To avoid serving
- * stale collection data, this object watches the /collections znode, which will
- * change if a collection is added or removed.
- */
- static final class PagedCollectionSupport implements Watcher, Comparator<String>, OnReconnect {
-
- // this is the full merged list of collections from ZooKeeper
- private List<String> cachedCollections;
-
- /**
- * If the list of collections changes, mark the cache as stale.
- */
- @Override
- public void process(WatchedEvent event) {
- // session events are not change events, and do not remove the watcher
- if (Event.EventType.None.equals(event.getType())) {
- return;
- }
- synchronized (this) {
- cachedCollections = null;
- }
- }
-
- /**
- * Create a merged view of all collections (internal from /clusterstate.json and external from /collections/?/state.json
- */
- private synchronized List<String> getCollections(SolrZkClient zkClient) throws KeeperException, InterruptedException {
- if (cachedCollections == null) {
- // cache is stale, rebuild the full list ...
- cachedCollections = new ArrayList<String>();
-
- List<String> fromZk = zkClient.getChildren("/collections", this, true);
- if (fromZk != null)
- cachedCollections.addAll(fromZk);
-
- // sort the final merged set of collections
- Collections.sort(cachedCollections, this);
- }
-
- return cachedCollections;
- }
-
- /**
- * Gets the requested page of collections after applying filters and offsets.
- */
- public PageOfCollections fetchPage(PageOfCollections page, SolrZkClient zkClient)
- throws KeeperException, InterruptedException {
-
-
- List<String> children = getCollections(zkClient);
- page.selected = children; // start with the page being the full list
-
- // activate paging (if disabled) for large collection sets
- if (page.start == 0 && page.rows == -1 && page.filter == null && children.size() > 10) {
- page.rows = 20;
- page.start = 0;
- }
-
- // apply the name filter if supplied (we don't need to pull state
- // data from ZK to do name filtering
- if (page.filterType == FilterType.name && page.filter != null)
- children = page.applyNameFilter(children);
-
- // a little hacky ... we can't select the page when filtering by
- // status until reading all status objects from ZK
- if (page.filterType != FilterType.status)
- page.selectPage(children);
-
- return page;
- }
-
- @Override
- public int compare(String left, String right) {
- if (left == null)
- return -1;
-
- if (left.equals(right))
- return 0;
-
- // sort lexically unless the two collection names start with the same base prefix
- // and end in a number (which is a common enough naming scheme to have direct
- // support for it)
- Matcher leftMatcher = endsWithDigits.matcher(left);
- if (leftMatcher.matches()) {
- Matcher rightMatcher = endsWithDigits.matcher(right);
- if (rightMatcher.matches()) {
- String leftGroup1 = leftMatcher.group(1);
- String rightGroup1 = rightMatcher.group(1);
- if (leftGroup1.equals(rightGroup1)) {
- // both start with the same prefix ... compare indexes
- // using longs here as we don't know how long the 2nd group is
- int leftGroup2 = Integer.parseInt(leftMatcher.group(2));
- int rightGroup2 = Integer.parseInt(rightMatcher.group(2));
- return (leftGroup2 > rightGroup2) ? 1 : ((leftGroup2 == rightGroup2) ? 0 : -1);
- }
- }
- }
- return left.compareTo(right);
- }
-
- /**
- * Called after a ZooKeeper session expiration occurs
- */
- @Override
- public void command() {
- // we need to re-establish the watcher on the collections list after session expires
- synchronized (this) {
- cachedCollections = null;
- }
- }
- }
-
- private PagedCollectionSupport pagingSupport;
-
- @Override
- public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
- final SolrParams params = req.getParams();
- Map<String, String> map = new HashMap<>(1);
- map.put(WT, "raw");
- map.put(OMIT_HEADER, "true");
- req.setParams(SolrParams.wrapDefaults(new MapSolrParams(map), params));
- synchronized (this) {
- if (pagingSupport == null) {
- pagingSupport = new PagedCollectionSupport();
- ZkController zkController = cores.getZkController();
- if (zkController != null) {
- // get notified when the ZK session expires (so we can clear the cached collections and rebuild)
- zkController.addOnReconnectListener(pagingSupport);
- }
- }
- }
-
- String path = params.get(PATH);
- String addr = params.get("addr");
-
- if (addr != null && addr.length() == 0) {
- addr = null;
- }
-
- String detailS = params.get("detail");
- boolean detail = detailS != null && detailS.equals("true");
-
- String dumpS = params.get("dump");
- boolean dump = dumpS != null && dumpS.equals("true");
-
- int start = params.getInt("start", 0);
- int rows = params.getInt("rows", -1);
-
- String filterType = params.get("filterType");
- if (filterType != null) {
- filterType = filterType.trim().toLowerCase(Locale.ROOT);
- if (filterType.length() == 0)
- filterType = null;
- }
- FilterType type = (filterType != null) ? FilterType.valueOf(filterType) : FilterType.none;
-
- String filter = (type != FilterType.none) ? params.get("filter") : null;
- if (filter != null) {
- filter = filter.trim();
- if (filter.length() == 0)
- filter = null;
- }
-
- ZKPrinter printer = new ZKPrinter(cores.getZkController(), addr);
- printer.detail = detail;
- printer.dump = dump;
- boolean isGraphView = "graph".equals(params.get("view"));
- printer.page = (isGraphView && "/clusterstate.json".equals(path))
- ? new PageOfCollections(start, rows, type, filter) : null;
- printer.pagingSupport = pagingSupport;
-
- try {
- printer.print(path);
- } finally {
- printer.close();
- }
- rsp.getValues().add(RawResponseWriter.CONTENT,printer);
- }
-
- //--------------------------------------------------------------------------------------
- //
- //--------------------------------------------------------------------------------------
-
- static class ZKPrinter implements ContentStream {
- static boolean FULLPATH_DEFAULT = false;
-
- boolean indent = true;
- boolean fullpath = FULLPATH_DEFAULT;
- boolean detail = false;
- boolean dump = false;
-
- String addr; // the address passed to us
- String keeperAddr; // the address we're connected to
-
- boolean doClose; // close the client after done if we opened it
-
- final BAOS baos = new BAOS();
- final Writer out = new OutputStreamWriter(baos, StandardCharsets.UTF_8);
- SolrZkClient zkClient;
-
- int level;
- int maxData = 95;
-
- PageOfCollections page;
- PagedCollectionSupport pagingSupport;
- ZkController zkController;
-
- public ZKPrinter(ZkController controller, String addr) throws IOException {
- this.zkController = controller;
- this.addr = addr;
-
- if (addr == null) {
- if (controller != null) {
- // this core is zk enabled
- keeperAddr = controller.getZkServerAddress();
- zkClient = controller.getZkClient();
- if (zkClient != null && zkClient.isConnected()) {
- return;
- } else {
- // try a different client with this address
- addr = keeperAddr;
- }
- }
- }
-
- keeperAddr = addr;
- if (addr == null) {
- writeError(404, "Zookeeper is not configured for this Solr Core. Please try connecting to an alternate zookeeper address.");
- return;
- }
-
- try {
- zkClient = new SolrZkClient(addr, 10000);
- doClose = true;
- } catch (Exception e) {
- writeError(503, "Could not connect to zookeeper at '" + addr + "'\"");
- zkClient = null;
- return;
- }
-
- }
-
- public void close() {
- if (doClose) {
- zkClient.close();
- }
- try {
- out.flush();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- // main entry point
- void print(String path) throws IOException {
- if (zkClient == null) {
- return;
- }
-
- // normalize path
- if (path == null) {
- path = "/";
- } else {
- path = path.trim();
- if (path.length() == 0) {
- path = "/";
- }
- }
-
- if (path.endsWith("/") && path.length() > 1) {
- path = path.substring(0, path.length() - 1);
- }
-
- int idx = path.lastIndexOf('/');
- String parent = idx >= 0 ? path.substring(0, idx) : path;
- if (parent.length() == 0) {
- parent = "/";
- }
-
- CharArr chars = new CharArr();
- JSONWriter json = new JSONWriter(chars, 2);
- json.startObject();
-
- if (detail) {
- if (!printZnode(json, path)) {
- return;
- }
- json.writeValueSeparator();
- }
-
- json.writeString("tree");
- json.writeNameSeparator();
- json.startArray();
- if (!printTree(json, path)) {
- return; // there was an error
- }
- json.endArray();
- json.endObject();
- out.write(chars.toString());
- }
-
- void writeError(int code, String msg) throws IOException {
- throw new SolrException(ErrorCode.getErrorCode(code), msg);
- /*response.setStatus(code);
-
- CharArr chars = new CharArr();
- JSONWriter w = new JSONWriter(chars, 2);
- w.startObject();
- w.indent();
- w.writeString("status");
- w.writeNameSeparator();
- w.write(code);
- w.writeValueSeparator();
- w.indent();
- w.writeString("error");
- w.writeNameSeparator();
- w.writeString(msg);
- w.endObject();
-
- out.write(chars.toString());*/
- }
-
-
- boolean printTree(JSONWriter json, String path) throws IOException {
- String label = path;
- if (!fullpath) {
- int idx = path.lastIndexOf('/');
- label = idx > 0 ? path.substring(idx + 1) : path;
- }
- json.startObject();
- //writeKeyValue(json, "data", label, true );
- json.writeString("data");
- json.writeNameSeparator();
-
- json.startObject();
- writeKeyValue(json, "title", label, true);
- json.writeValueSeparator();
- json.writeString("attr");
- json.writeNameSeparator();
- json.startObject();
- writeKeyValue(json, "href", "admin/zookeeper?detail=true&path=" + URLEncoder.encode(path, "UTF-8"), true);
- json.endObject();
- json.endObject();
-
- Stat stat = new Stat();
- try {
- // Trickily, the call to zkClient.getData fills in the stat variable
- byte[] data = zkClient.getData(path, null, stat, true);
-
- if (stat.getEphemeralOwner() != 0) {
- writeKeyValue(json, "ephemeral", true, false);
- writeKeyValue(json, "version", stat.getVersion(), false);
- }
-
- if (dump) {
- json.writeValueSeparator();
- printZnode(json, path);
- }
-
- } catch (IllegalArgumentException e) {
- // path doesn't exist (must have been removed)
- writeKeyValue(json, "warning", "(path gone)", false);
- } catch (KeeperException e) {
- writeKeyValue(json, "warning", e.toString(), false);
- log.warn("Keeper Exception", e);
- } catch (InterruptedException e) {
- writeKeyValue(json, "warning", e.toString(), false);
- log.warn("InterruptedException", e);
- }
-
- if (stat.getNumChildren() > 0) {
- json.writeValueSeparator();
- if (indent) {
- json.indent();
- }
- json.writeString("children");
- json.writeNameSeparator();
- json.startArray();
-
- try {
- List<String> children = zkClient.getChildren(path, null, true);
- java.util.Collections.sort(children);
-
- boolean first = true;
- for (String child : children) {
- if (!first) {
- json.writeValueSeparator();
- }
-
- String childPath = path + (path.endsWith("/") ? "" : "/") + child;
- if (!printTree(json, childPath)) {
- return false;
- }
- first = false;
- }
- } catch (KeeperException e) {
- writeError(500, e.toString());
- return false;
- } catch (InterruptedException e) {
- writeError(500, e.toString());
- return false;
- } catch (IllegalArgumentException e) {
- // path doesn't exist (must have been removed)
- json.writeString("(children gone)");
- }
-
- json.endArray();
- }
-
- json.endObject();
- return true;
- }
-
- String time(long ms) {
- return (new Date(ms)).toString() + " (" + ms + ")";
- }
-
- public void writeKeyValue(JSONWriter json, String k, Object v, boolean isFirst) {
- if (!isFirst) {
- json.writeValueSeparator();
- }
- if (indent) {
- json.indent();
- }
- json.writeString(k);
- json.writeNameSeparator();
- json.write(v);
- }
-
- @SuppressWarnings("unchecked")
- boolean printZnode(JSONWriter json, String path) throws IOException {
- try {
- String dataStr = null;
- String dataStrErr = null;
- Stat stat = new Stat();
- // Trickily, the call to zkClient.getData fills in the stat variable
- byte[] data = zkClient.getData(path, null, stat, true);
- if (null != data) {
- try {
- dataStr = (new BytesRef(data)).utf8ToString();
- } catch (Exception e) {
- dataStrErr = "data is not parsable as a utf8 String: " + e.toString();
- }
- }
- // support paging of the collections graph view (in case there are many collections)
- if (page != null) {
- // we've already pulled the data for /clusterstate.json from ZooKeeper above,
- // but it needs to be parsed into a map so we can lookup collection states before
- // trying to find them in the /collections/?/state.json znode
- Map<String, Object> clusterstateJsonMap = null;
- if (dataStr != null) {
- try {
- clusterstateJsonMap = (Map<String, Object>) ObjectBuilder.fromJSON(dataStr);
- } catch (Exception e) {
- throw new SolrException(ErrorCode.SERVER_ERROR,
- "Failed to parse /clusterstate.json from ZooKeeper due to: " + e, e);
- }
- } else {
- clusterstateJsonMap = Utils.makeMap();
- }
-
- // fetch the requested page of collections and then retrieve the state for each
- page = pagingSupport.fetchPage(page, zkClient);
- // keep track of how many collections match the filter
- boolean applyStatusFilter =
- (page.filterType == FilterType.status && page.filter != null);
- List<String> matchesStatusFilter = applyStatusFilter ? new ArrayList<String>() : null;
- Set<String> liveNodes = applyStatusFilter ?
- zkController.getZkStateReader().getClusterState().getLiveNodes() : null;
-
- SortedMap<String, Object> collectionStates = new TreeMap<String, Object>(pagingSupport);
- for (String collection : page.selected) {
- Object collectionState = clusterstateJsonMap.get(collection);
- if (collectionState != null) {
- // collection state was in /clusterstate.json
- if (applyStatusFilter) {
- // verify this collection matches the status filter
- if (page.matchesStatusFilter((Map<String, Object>) collectionState, liveNodes)) {
- matchesStatusFilter.add(collection);
- collectionStates.put(collection, collectionState);
- }
- } else {
- collectionStates.put(collection, collectionState);
- }
- } else {
- // looks like an external collection ...
- String collStatePath = String.format(Locale.ROOT, "/collections/%s/state.json", collection);
- String childDataStr = null;
- try {
- byte[] childData = zkClient.getData(collStatePath, null, null, true);
- if (childData != null)
- childDataStr = (new BytesRef(childData)).utf8ToString();
- } catch (KeeperException.NoNodeException nne) {
- log.warn("State for collection " + collection +
- " not found in /clusterstate.json or /collections/" + collection + "/state.json!");
- } catch (Exception childErr) {
- log.error("Failed to get " + collStatePath + " due to: " + childErr);
- }
-
- if (childDataStr != null) {
- Map<String, Object> extColl = (Map<String, Object>) ObjectBuilder.fromJSON(childDataStr);
- collectionState = extColl.get(collection);
-
- if (applyStatusFilter) {
- // verify this collection matches the filtered state
- if (page.matchesStatusFilter((Map<String, Object>) collectionState, liveNodes)) {
- matchesStatusFilter.add(collection);
- collectionStates.put(collection, collectionState);
- }
- } else {
- collectionStates.put(collection, collectionState);
- }
- }
- }
- }
-
- if (applyStatusFilter) {
- // update the paged navigation info after applying the status filter
- page.selectPage(matchesStatusFilter);
-
- // rebuild the Map of state data
- SortedMap<String, Object> map = new TreeMap<String, Object>(pagingSupport);
- for (String next : page.selected)
- map.put(next, collectionStates.get(next));
- collectionStates = map;
- }
-
- if (collectionStates != null) {
- CharArr out = new CharArr();
- new JSONWriter(out, 2).write(collectionStates);
- dataStr = out.toString();
- }
- }
-
- json.writeString("znode");
- json.writeNameSeparator();
- json.startObject();
-
- writeKeyValue(json, PATH, path, true);
-
- json.writeValueSeparator();
- json.writeString("prop");
- json.writeNameSeparator();
- json.startObject();
- writeKeyValue(json, "version", stat.getVersion(), true);
- writeKeyValue(json, "aversion", stat.getAversion(), false);
- writeKeyValue(json, "children_count", stat.getNumChildren(), false);
- writeKeyValue(json, "ctime", time(stat.getCtime()), false);
- writeKeyValue(json, "cversion", stat.getCversion(), false);
- writeKeyValue(json, "czxid", stat.getCzxid(), false);
- writeKeyValue(json, "ephemeralOwner", stat.getEphemeralOwner(), false);
- writeKeyValue(json, "mtime", time(stat.getMtime()), false);
- writeKeyValue(json, "mzxid", stat.getMzxid(), false);
- writeKeyValue(json, "pzxid", stat.getPzxid(), false);
- writeKeyValue(json, "dataLength", stat.getDataLength(), false);
- if (null != dataStrErr) {
- writeKeyValue(json, "dataNote", dataStrErr, false);
- }
- json.endObject();
-
- if (null != dataStr) {
- writeKeyValue(json, "data", dataStr, false);
- }
-
- if (page != null) {
- writeKeyValue(json, "paging", page.getPagingHeader(), false);
- }
-
- json.endObject();
- } catch (KeeperException e) {
- writeError(500, e.toString());
- return false;
- } catch (InterruptedException e) {
- writeError(500, e.toString());
- return false;
- }
- return true;
- }
-
- /* @Override
- public void write(OutputStream os) throws IOException {
- ByteBuffer bytes = baos.getByteBuffer();
- os.write(bytes.array(),0,bytes.limit());
- }
-*/
- @Override
- public String getName() {
- return null;
- }
-
- @Override
- public String getSourceInfo() {
- return null;
- }
-
- @Override
- public String getContentType() {
- return JSONResponseWriter.CONTENT_TYPE_JSON_UTF8;
- }
-
- @Override
- public Long getSize() {
- return null;
- }
-
- @Override
- public InputStream getStream() throws IOException {
- return new ByteBufferInputStream(baos.getByteBuffer());
- }
-
- @Override
- public Reader getReader() throws IOException {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java
deleted file mode 100644
index 8842437..0000000
--- a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.admin;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.io.Writer;
-import java.lang.invoke.MethodHandles;
-import java.net.Socket;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.handler.RequestHandlerBase;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Zookeeper Status handler, talks to ZK using sockets and four-letter words
- *
- * @since solr 7.5
- */
-public final class ZookeeperStatusHandler extends RequestHandlerBase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final int ZOOKEEPER_DEFAULT_PORT = 2181;
- private static final String STATUS_RED = "red";
- private static final String STATUS_GREEN = "green";
- private static final String STATUS_YELLOW = "yellow";
- private static final String STATUS_NA = "N/A";
- private CoreContainer cores;
-
- public ZookeeperStatusHandler(CoreContainer cc) {
- this.cores = cc;
- }
-
- @Override
- public String getDescription() {
- return "Fetch Zookeeper status";
- }
-
- @Override
- public Category getCategory() {
- return Category.ADMIN;
- }
-
- @Override
- public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
- final SolrParams params = req.getParams();
- Map<String, String> map = new HashMap<>(1);
- NamedList values = rsp.getValues();
- values.add("zkStatus", getZkStatus(cores.getZkController().getZkServerAddress()));
- }
-
- /*
- Gets all info from ZK API and returns as a map
- */
- protected Map<String, Object> getZkStatus(String zkHost) {
- Map<String, Object> zkStatus = new HashMap<>();
- List<String> zookeepers = Arrays.asList(zkHost.split("/")[0].split(","));
- List<Object> details = new ArrayList<>();
- int numOk = 0;
- String status = STATUS_NA;
- int standalone = 0;
- int followers = 0;
- int reportedFollowers = 0;
- int leaders = 0;
- List<String> errors = new ArrayList<>();
- for (String zk : zookeepers) {
- try {
- Map<String, Object> stat = monitorZookeeper(zk);
- details.add(stat);
- if ("true".equals(String.valueOf(stat.get("ok")))) {
- numOk++;
- }
- String state = String.valueOf(stat.get("zk_server_state"));
- if ("follower".equals(state)) {
- followers++;
- } else if ("leader".equals(state)) {
- leaders++;
- reportedFollowers = Integer.parseInt(String.valueOf(stat.get("zk_followers")));
- } else if ("standalone".equals(state)) {
- standalone++;
- }
- } catch (SolrException se) {
- log.warn("Failed talking to zookeeper" + zk, se);
- errors.add(se.getMessage());
- Map<String, Object> stat = new HashMap<>();
- stat.put("host", zk);
- stat.put("ok", false);
- details.add(stat);
- }
- }
- zkStatus.put("ensembleSize", zookeepers.size());
- zkStatus.put("zkHost", zkHost);
- zkStatus.put("details", details);
- if (followers+leaders > 0 && standalone > 0) {
- status = STATUS_RED;
- errors.add("The zk nodes do not agree on their mode, check details");
- }
- if (standalone > 1) {
- status = STATUS_RED;
- errors.add("Only one zk allowed in standalone mode");
- }
- if (leaders > 1) {
- zkStatus.put("mode", "ensemble");
- status = STATUS_RED;
- errors.add("Only one leader allowed, got " + leaders);
- }
- if (followers > 0 && leaders == 0) {
- zkStatus.put("mode", "ensemble");
- status = STATUS_RED;
- errors.add("We do not have a leader");
- }
- if (leaders > 0 && followers != reportedFollowers) {
- zkStatus.put("mode", "ensemble");
- status = STATUS_RED;
- errors.add("Leader reports " + reportedFollowers + " followers, but we only found " + followers +
- ". Please check zkHost configuration");
- }
- if (followers+leaders == 0 && standalone == 1) {
- zkStatus.put("mode", "standalone");
- }
- if (followers+leaders > 0 && (zookeepers.size())%2 == 0) {
- if (!STATUS_RED.equals(status)) {
- status = STATUS_YELLOW;
- }
- errors.add("We have an even number of zookeepers which is not recommended");
- }
- if (followers+leaders > 0 && standalone == 0) {
- zkStatus.put("mode", "ensemble");
- }
- if (status.equals(STATUS_NA)) {
- if (numOk == zookeepers.size()) {
- status = STATUS_GREEN;
- } else if (numOk < zookeepers.size() && numOk > zookeepers.size() / 2) {
- status = STATUS_YELLOW;
- errors.add("Some zookeepers are down: " + numOk + "/" + zookeepers.size());
- } else {
- status = STATUS_RED;
- errors.add("Mismatch in number of zookeeper nodes live. numOK=" + numOk + ", expected " + zookeepers.size());
- }
- }
- zkStatus.put("status", status);
- if (!errors.isEmpty()) {
- zkStatus.put("errors", errors);
- }
- return zkStatus;
- }
-
- private Map<String, Object> monitorZookeeper(String zkHostPort) {
- List<String> lines = getZkRawResponse(zkHostPort, "mntr");
- Map<String, Object> obj = new HashMap<>();
- obj.put("host", zkHostPort);
- obj.put("ok", "imok".equals(getZkRawResponse(zkHostPort, "ruok").get(0)));
- for (String line : lines) {
- obj.put(line.split("\t")[0], line.split("\t")[1]);
- }
- lines = getZkRawResponse(zkHostPort, "conf");
- for (String line : lines) {
- obj.put(line.split("=")[0], line.split("=")[1]);
- }
- return obj;
- }
-
- /**
- * Sends a four-letter-word command to one particular Zookeeper server and returns the response as list of strings
- * @param zkHostPort the host:port for one zookeeper server to access
- * @param fourLetterWordCommand the custom 4-letter command to send to Zookeeper
- * @return a list of lines returned from Zookeeper
- */
- private List<String> getZkRawResponse(String zkHostPort, String fourLetterWordCommand) {
- String[] hostPort = zkHostPort.split(":");
- String host = hostPort[0];
- int port = ZOOKEEPER_DEFAULT_PORT;
- if (hostPort.length > 1) {
- port = Integer.parseInt(hostPort[1]);
- }
- try (
- Socket socket = new Socket(host, port);
- Writer writer = new OutputStreamWriter(socket.getOutputStream(), "utf-8");
- PrintWriter out = new PrintWriter(writer, true);
- BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream(), "utf-8"));) {
- out.println(fourLetterWordCommand);
- List<String> response = in.lines().collect(Collectors.toList());
- log.debug("Got response from ZK on host {} and port {}: {}", host, port, response);
- if (response == null || response.isEmpty()) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Empty response from Zookeeper " + zkHostPort);
- }
- return response;
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Failed talking to Zookeeper " + zkHostPort, e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/handler/admin/package-info.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/package-info.java b/solr/core/src/java/org/apache/solr/handler/admin/package-info.java
deleted file mode 100644
index bce6448..0000000
--- a/solr/core/src/java/org/apache/solr/handler/admin/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * {@link org.apache.solr.request.SolrRequestHandler} implementations for powering he Solr Admin UI
- */
-package org.apache.solr.handler.admin;
-
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java
deleted file mode 100644
index 1f398a9..0000000
--- a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.component;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.lucene.search.Query;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.common.util.SuppressForbidden;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.search.DocList;
-import org.apache.solr.search.QueryParsing;
-import org.apache.solr.search.facet.FacetDebugInfo;
-import org.apache.solr.util.SolrPluginUtils;
-
-import static org.apache.solr.common.params.CommonParams.FQ;
-import static org.apache.solr.common.params.CommonParams.JSON;
-
-/**
- * Adds debugging information to a request.
- *
- *
- * @since solr 1.3
- */
-public class DebugComponent extends SearchComponent
-{
- public static final String COMPONENT_NAME = "debug";
-
- /**
- * A counter to ensure that no RID is equal, even if they fall in the same millisecond
- */
- private static final AtomicLong ridCounter = new AtomicLong();
-
- /**
- * Map containing all the possible stages as key and
- * the corresponding readable purpose as value
- */
- private static final Map<Integer, String> stages;
-
- static {
- Map<Integer, String> map = new TreeMap<>();
- map.put(ResponseBuilder.STAGE_START, "START");
- map.put(ResponseBuilder.STAGE_PARSE_QUERY, "PARSE_QUERY");
- map.put(ResponseBuilder.STAGE_TOP_GROUPS, "TOP_GROUPS");
- map.put(ResponseBuilder.STAGE_EXECUTE_QUERY, "EXECUTE_QUERY");
- map.put(ResponseBuilder.STAGE_GET_FIELDS, "GET_FIELDS");
- map.put(ResponseBuilder.STAGE_DONE, "DONE");
- stages = Collections.unmodifiableMap(map);
- }
-
- @Override
- public void prepare(ResponseBuilder rb) throws IOException
- {
- if(rb.isDebugTrack() && rb.isDistrib) {
- rb.setNeedDocList(true);
- doDebugTrack(rb);
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void process(ResponseBuilder rb) throws IOException
- {
- if( rb.isDebug() ) {
- DocList results = null;
- //some internal grouping requests won't have results value set
- if(rb.getResults() != null) {
- results = rb.getResults().docList;
- }
-
- NamedList stdinfo = SolrPluginUtils.doStandardDebug( rb.req,
- rb.getQueryString(), rb.wrap(rb.getQuery()), results, rb.isDebugQuery(), rb.isDebugResults());
-
- NamedList info = rb.getDebugInfo();
- if( info == null ) {
- rb.setDebugInfo( stdinfo );
- info = stdinfo;
- }
- else {
- info.addAll( stdinfo );
- }
-
- FacetDebugInfo fdebug = (FacetDebugInfo)(rb.req.getContext().get("FacetDebugInfo"));
- if (fdebug != null) {
- info.add("facet-trace", fdebug.getFacetDebugInfo());
- }
-
- fdebug = (FacetDebugInfo)(rb.req.getContext().get("FacetDebugInfo-nonJson"));
- if (fdebug != null) {
- info.add("facet-debug", fdebug.getFacetDebugInfo());
- }
-
- if (rb.req.getJSON() != null) {
- info.add(JSON, rb.req.getJSON());
- }
-
- if (rb.isDebugQuery() && rb.getQparser() != null) {
- rb.getQparser().addDebugInfo(rb.getDebugInfo());
- }
-
- if (null != rb.getDebugInfo() ) {
- if (rb.isDebugQuery() && null != rb.getFilters() ) {
- info.add("filter_queries",rb.req.getParams().getParams(FQ));
- List<String> fqs = new ArrayList<>(rb.getFilters().size());
- for (Query fq : rb.getFilters()) {
- fqs.add(QueryParsing.toString(fq, rb.req.getSchema()));
- }
- info.add("parsed_filter_queries",fqs);
- }
-
- // Add this directly here?
- rb.rsp.add("debug", rb.getDebugInfo() );
- }
- }
- }
-
-
- private void doDebugTrack(ResponseBuilder rb) {
- SolrQueryRequest req = rb.req;
- String rid = req.getParams().get(CommonParams.REQUEST_ID);
- if(rid == null || "".equals(rid)) {
- rid = generateRid(rb);
- ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
- params.add(CommonParams.REQUEST_ID, rid);//add rid to the request so that shards see it
- req.setParams(params);
- }
- rb.addDebug(rid, "track", CommonParams.REQUEST_ID);//to see it in the response
- rb.rsp.addToLog(CommonParams.REQUEST_ID, rid); //to see it in the logs of the landing core
-
- }
-
- @SuppressForbidden(reason = "Need currentTimeMillis, only used for naming")
- private String generateRid(ResponseBuilder rb) {
- String hostName = rb.req.getCore().getCoreContainer().getHostName();
- return hostName + "-" + rb.req.getCore().getName() + "-" + System.currentTimeMillis() + "-" + ridCounter.getAndIncrement();
- }
-
- @Override
- public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
- if (!rb.isDebug()) return;
-
- // Turn on debug to get explain only when retrieving fields
- if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
- sreq.purpose |= ShardRequest.PURPOSE_GET_DEBUG;
- if (rb.isDebugAll()) {
- sreq.params.set(CommonParams.DEBUG_QUERY, "true");
- } else {
- if (rb.isDebugQuery()){
- sreq.params.add(CommonParams.DEBUG, CommonParams.QUERY);
- }
- if (rb.isDebugResults()){
- sreq.params.add(CommonParams.DEBUG, CommonParams.RESULTS);
- }
- }
- } else {
- sreq.params.set(CommonParams.DEBUG_QUERY, "false");
- sreq.params.set(CommonParams.DEBUG, "false");
- }
- if (rb.isDebugTimings()) {
- sreq.params.add(CommonParams.DEBUG, CommonParams.TIMING);
- }
- if (rb.isDebugTrack()) {
- sreq.params.add(CommonParams.DEBUG, CommonParams.TRACK);
- sreq.params.set(CommonParams.REQUEST_ID, rb.req.getParams().get(CommonParams.REQUEST_ID));
- sreq.params.set(CommonParams.REQUEST_PURPOSE, SolrPluginUtils.getRequestPurpose(sreq.purpose));
- }
- }
-
- @Override
- public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
- if (rb.isDebugTrack() && rb.isDistrib && !rb.finished.isEmpty()) {
- @SuppressWarnings("unchecked")
- NamedList<Object> stageList = (NamedList<Object>) ((NamedList<Object>)rb.getDebugInfo().get("track")).get(stages.get(rb.stage));
- if(stageList == null) {
- stageList = new SimpleOrderedMap<>();
- rb.addDebug(stageList, "track", stages.get(rb.stage));
- }
- for(ShardResponse response: sreq.responses) {
- stageList.add(response.getShard(), getTrackResponse(response));
- }
- }
- }
-
- private final static Set<String> EXCLUDE_SET = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("explain")));
-
- @Override
- public void finishStage(ResponseBuilder rb) {
- if (rb.isDebug() && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
- NamedList<Object> info = rb.getDebugInfo();
- NamedList<Object> explain = new SimpleOrderedMap<>();
-
- Map.Entry<String, Object>[] arr = new NamedList.NamedListEntry[rb.resultIds.size()];
- // Will be set to true if there is at least one response with PURPOSE_GET_DEBUG
- boolean hasGetDebugResponses = false;
-
- for (ShardRequest sreq : rb.finished) {
- for (ShardResponse srsp : sreq.responses) {
- if (srsp.getException() != null) {
- // can't expect the debug content if there was an exception for this request
- // this should only happen when using shards.tolerant=true
- continue;
- }
- NamedList sdebug = (NamedList)srsp.getSolrResponse().getResponse().get("debug");
- info = (NamedList)merge(sdebug, info, EXCLUDE_SET);
- if ((sreq.purpose & ShardRequest.PURPOSE_GET_DEBUG) != 0) {
- hasGetDebugResponses = true;
- if (rb.isDebugResults()) {
- NamedList sexplain = (NamedList)sdebug.get("explain");
- SolrPluginUtils.copyNamedListIntoArrayByDocPosInResponse(sexplain, rb.resultIds, arr);
- }
- }
- }
- }
-
- if (rb.isDebugResults()) {
- explain = SolrPluginUtils.removeNulls(arr, new SimpleOrderedMap<>());
- }
-
- if (!hasGetDebugResponses) {
- if (info == null) {
- info = new SimpleOrderedMap<>();
- }
- // No responses were received from shards. Show local query info.
- SolrPluginUtils.doStandardQueryDebug(
- rb.req, rb.getQueryString(), rb.wrap(rb.getQuery()), rb.isDebugQuery(), info);
- if (rb.isDebugQuery() && rb.getQparser() != null) {
- rb.getQparser().addDebugInfo(info);
- }
- }
- if (rb.isDebugResults()) {
- int idx = info.indexOf("explain",0);
- if (idx>=0) {
- info.setVal(idx, explain);
- } else {
- info.add("explain", explain);
- }
- }
-
- rb.setDebugInfo(info);
- rb.rsp.add("debug", rb.getDebugInfo() );
- }
-
- }
-
-
- private NamedList<String> getTrackResponse(ShardResponse shardResponse) {
- NamedList<String> namedList = new SimpleOrderedMap<>();
- if (shardResponse.getException() != null) {
- namedList.add("Exception", shardResponse.getException().getMessage());
- return namedList;
- }
- NamedList<Object> responseNL = shardResponse.getSolrResponse().getResponse();
- @SuppressWarnings("unchecked")
- NamedList<Object> responseHeader = (NamedList<Object>)responseNL.get("responseHeader");
- if(responseHeader != null) {
- namedList.add("QTime", responseHeader.get("QTime").toString());
- }
- namedList.add("ElapsedTime", String.valueOf(shardResponse.getSolrResponse().getElapsedTime()));
- namedList.add("RequestPurpose", shardResponse.getShardRequest().params.get(CommonParams.REQUEST_PURPOSE));
- SolrDocumentList docList = (SolrDocumentList)shardResponse.getSolrResponse().getResponse().get("response");
- if(docList != null) {
- namedList.add("NumFound", String.valueOf(docList.getNumFound()));
- }
- namedList.add("Response", String.valueOf(responseNL));
- return namedList;
- }
-
- protected Object merge(Object source, Object dest, Set<String> exclude) {
- if (source == null) return dest;
- if (dest == null) {
- if (source instanceof NamedList) {
- dest = source instanceof SimpleOrderedMap ? new SimpleOrderedMap() : new NamedList();
- } else {
- return source;
- }
- } else {
-
- if (dest instanceof Collection) {
- // merge as Set
- if (!(dest instanceof Set)) {
- dest = new LinkedHashSet<>((Collection<?>) dest);
- }
- if (source instanceof Collection) {
- ((Collection)dest).addAll((Collection)source);
- } else {
- ((Collection)dest).add(source);
- }
- return dest;
- } else if (source instanceof Number) {
- if (dest instanceof Number) {
- if (source instanceof Double || dest instanceof Double) {
- return ((Number)source).doubleValue() + ((Number)dest).doubleValue();
- }
- return ((Number)source).longValue() + ((Number)dest).longValue();
- }
- // fall through
- } else if (source instanceof String) {
- if (source.equals(dest)) {
- return dest;
- }
- // fall through
- }
- }
-
-
- if (source instanceof NamedList && dest instanceof NamedList) {
- NamedList<Object> tmp = new NamedList<>();
- @SuppressWarnings("unchecked")
- NamedList<Object> sl = (NamedList<Object>)source;
- @SuppressWarnings("unchecked")
- NamedList<Object> dl = (NamedList<Object>)dest;
- for (int i=0; i<sl.size(); i++) {
- String skey = sl.getName(i);
- if (exclude.contains(skey)) continue;
- Object sval = sl.getVal(i);
- int didx = -1;
-
- // optimize case where elements are in same position
- if (i < dl.size()) {
- String dkey = dl.getName(i);
- if (skey == dkey || (skey!=null && skey.equals(dkey))) {
- didx = i;
- }
- }
-
- if (didx == -1) {
- didx = dl.indexOf(skey, 0);
- }
-
- if (didx == -1) {
- tmp.add(skey, merge(sval, null, Collections.emptySet()));
- } else {
- dl.setVal(didx, merge(sval, dl.getVal(didx), Collections.emptySet()));
- }
- }
- dl.addAll(tmp);
- return dl;
- }
-
- // only add to list if JSON is different
- if (source.equals(dest)) return source;
-
- // merge unlike elements in a list
- List<Object> t = new ArrayList<>();
- t.add(dest);
- t.add(source);
- return t;
- }
-
-
-
- /////////////////////////////////////////////
- /// SolrInfoBean
- ////////////////////////////////////////////
-
- @Override
- public String getDescription() {
- return "Debug Information";
- }
-
- @Override
- public Category getCategory() {
- return Category.OTHER;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0ae21ad0/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
deleted file mode 100644
index 2cbe703..0000000
--- a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
+++ /dev/null
@@ -1,828 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.component;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-
-import com.carrotsearch.hppc.IntHashSet;
-import com.carrotsearch.hppc.IntObjectHashMap;
-import com.carrotsearch.hppc.LongHashSet;
-import com.carrotsearch.hppc.LongObjectHashMap;
-import com.carrotsearch.hppc.LongObjectMap;
-import com.carrotsearch.hppc.cursors.IntObjectCursor;
-import com.carrotsearch.hppc.cursors.LongCursor;
-import com.carrotsearch.hppc.cursors.LongObjectCursor;
-import com.carrotsearch.hppc.cursors.ObjectCursor;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.FilterLeafReader;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiDocValues;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.OrdinalMap;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorable;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.TermInSetQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.TopDocsCollector;
-import org.apache.lucene.search.TopFieldCollector;
-import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.search.TotalHits;
-import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.CharsRefBuilder;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.LongValues;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.params.ExpandParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.core.PluginInfo;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.schema.FieldType;
-import org.apache.solr.schema.NumberType;
-import org.apache.solr.schema.SchemaField;
-import org.apache.solr.schema.StrField;
-import org.apache.solr.search.CollapsingQParserPlugin;
-import org.apache.solr.search.DocIterator;
-import org.apache.solr.search.DocList;
-import org.apache.solr.search.DocSlice;
-import org.apache.solr.search.QParser;
-import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.search.SortSpecParsing;
-import org.apache.solr.uninverting.UninvertingReader;
-import org.apache.solr.util.plugin.PluginInfoInitialized;
-import org.apache.solr.util.plugin.SolrCoreAware;
-
-/**
- * The ExpandComponent is designed to work with the CollapsingPostFilter.
- * The CollapsingPostFilter collapses a result set on a field.
- * <p>
- * The ExpandComponent expands the collapsed groups for a single page.
- * <p>
- * http parameters:
- * <p>
- * expand=true <br>
- * expand.rows=5 <br>
- * expand.sort=field asc|desc<br>
- * expand.q=*:* (optional, overrides the main query)<br>
- * expand.fq=type:child (optional, overrides the main filter queries)<br>
- * expand.field=field (mandatory if the not used with the CollapsingQParserPlugin)<br>
- */
-public class ExpandComponent extends SearchComponent implements PluginInfoInitialized, SolrCoreAware {
- public static final String COMPONENT_NAME = "expand";
- private static final int finishingStage = ResponseBuilder.STAGE_GET_FIELDS;
- private PluginInfo info = PluginInfo.EMPTY_INFO;
-
- @Override
- public void init(PluginInfo info) {
- this.info = info;
- }
-
- @Override
- public void prepare(ResponseBuilder rb) throws IOException {
- if (rb.req.getParams().getBool(ExpandParams.EXPAND, false)) {
- rb.doExpand = true;
- }
- }
-
- @Override
- public void inform(SolrCore core) {
-
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void process(ResponseBuilder rb) throws IOException {
-
- if (!rb.doExpand) {
- return;
- }
-
- SolrQueryRequest req = rb.req;
- SolrParams params = req.getParams();
-
- String field = params.get(ExpandParams.EXPAND_FIELD);
- String hint = null;
- if (field == null) {
- List<Query> filters = rb.getFilters();
- if (filters != null) {
- for (Query q : filters) {
- if (q instanceof CollapsingQParserPlugin.CollapsingPostFilter) {
- CollapsingQParserPlugin.CollapsingPostFilter cp = (CollapsingQParserPlugin.CollapsingPostFilter) q;
- field = cp.getField();
- hint = cp.hint;
- }
- }
- }
- }
-
- if (field == null) {
- throw new IOException("Expand field is null.");
- }
-
- String sortParam = params.get(ExpandParams.EXPAND_SORT);
- String[] fqs = params.getParams(ExpandParams.EXPAND_FQ);
- String qs = params.get(ExpandParams.EXPAND_Q);
- int limit = params.getInt(ExpandParams.EXPAND_ROWS, 5);
-
- Sort sort = null;
-
- if (sortParam != null) {
- sort = SortSpecParsing.parseSortSpec(sortParam, rb.req).getSort();
- }
-
- Query query;
- if (qs == null) {
- query = rb.getQuery();
- } else {
- try {
- QParser parser = QParser.getParser(qs, req);
- query = parser.getQuery();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- List<Query> newFilters = new ArrayList<>();
-
- if (fqs == null) {
- List<Query> filters = rb.getFilters();
- if (filters != null) {
- for (Query q : filters) {
- if (!(q instanceof CollapsingQParserPlugin.CollapsingPostFilter)) {
- newFilters.add(q);
- }
- }
- }
- } else {
- try {
- for (String fq : fqs) {
- if (fq != null && fq.trim().length() != 0 && !fq.equals("*:*")) {
- QParser fqp = QParser.getParser(fq, req);
- newFilters.add(fqp.getQuery());
- }
- }
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- SolrIndexSearcher searcher = req.getSearcher();
- LeafReader reader = searcher.getSlowAtomicReader();
-
- SchemaField schemaField = searcher.getSchema().getField(field);
- FieldType fieldType = schemaField.getType();
-
- SortedDocValues values = null;
- long nullValue = 0L;
-
- if(fieldType instanceof StrField) {
- //Get The Top Level SortedDocValues
- if(CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
- @SuppressWarnings("resource") LeafReader uninvertingReader = UninvertingReader.wrap(
- new ReaderWrapper(searcher.getSlowAtomicReader(), field),
- Collections.singletonMap(field, UninvertingReader.Type.SORTED)::get);
- values = uninvertingReader.getSortedDocValues(field);
- } else {
- values = DocValues.getSorted(reader, field);
- }
- } else {
- //Get the nullValue for the numeric collapse field
- String defaultValue = searcher.getSchema().getField(field).getDefaultValue();
-
- final NumberType numType = fieldType.getNumberType();
-
- // Since the expand component depends on the operation of the collapse component,
- // which validates that numeric field types are 32-bit,
- // we don't need to handle invalid 64-bit field types here.
- if (defaultValue != null) {
- if (numType == NumberType.INTEGER) {
- nullValue = Long.parseLong(defaultValue);
- } else if (numType == NumberType.FLOAT) {
- nullValue = Float.floatToIntBits(Float.parseFloat(defaultValue));
- }
- } else if (NumberType.FLOAT.equals(numType)) { // Integer case already handled by nullValue defaulting to 0
- nullValue = Float.floatToIntBits(0.0f);
- }
- }
-
- FixedBitSet groupBits = null;
- LongHashSet groupSet = null;
- DocList docList = rb.getResults().docList;
- IntHashSet collapsedSet = new IntHashSet(docList.size() * 2);
-
- //Gather the groups for the current page of documents
- DocIterator idit = docList.iterator();
- int[] globalDocs = new int[docList.size()];
- int docsIndex = -1;
- while (idit.hasNext()) {
- globalDocs[++docsIndex] = idit.nextDoc();
- }
-
- Arrays.sort(globalDocs);
- Query groupQuery = null;
-
- /*
- * This code gathers the group information for the current page.
- */
- List<LeafReaderContext> contexts = searcher.getTopReaderContext().leaves();
-
- if(contexts.size() == 0) {
- //When no context is available we can skip the expanding
- return;
- }
-
- int currentContext = 0;
- int currentDocBase = contexts.get(currentContext).docBase;
- int nextDocBase = (currentContext+1)<contexts.size() ? contexts.get(currentContext+1).docBase : Integer.MAX_VALUE;
- IntObjectHashMap<BytesRef> ordBytes = null;
- if(values != null) {
- groupBits = new FixedBitSet(values.getValueCount());
- OrdinalMap ordinalMap = null;
- SortedDocValues[] sortedDocValues = null;
- LongValues segmentOrdinalMap = null;
- SortedDocValues currentValues = null;
- if(values instanceof MultiDocValues.MultiSortedDocValues) {
- ordinalMap = ((MultiDocValues.MultiSortedDocValues)values).mapping;
- sortedDocValues = ((MultiDocValues.MultiSortedDocValues)values).values;
- currentValues = sortedDocValues[currentContext];
- segmentOrdinalMap = ordinalMap.getGlobalOrds(currentContext);
- }
- int count = 0;
-
- ordBytes = new IntObjectHashMap<>();
-
- for(int i=0; i<globalDocs.length; i++) {
- int globalDoc = globalDocs[i];
- while(globalDoc >= nextDocBase) {
- currentContext++;
- currentDocBase = contexts.get(currentContext).docBase;
- nextDocBase = (currentContext+1) < contexts.size() ? contexts.get(currentContext+1).docBase : Integer.MAX_VALUE;
- if(ordinalMap != null) {
- currentValues = sortedDocValues[currentContext];
- segmentOrdinalMap = ordinalMap.getGlobalOrds(currentContext);
- }
- }
-
- int contextDoc = globalDoc - currentDocBase;
- if(ordinalMap != null) {
- if (contextDoc > currentValues.docID()) {
- currentValues.advance(contextDoc);
- }
- if (contextDoc == currentValues.docID()) {
- int ord = currentValues.ordValue();
- ++count;
- BytesRef ref = currentValues.lookupOrd(ord);
- ord = (int)segmentOrdinalMap.get(ord);
- ordBytes.put(ord, BytesRef.deepCopyOf(ref));
- groupBits.set(ord);
- collapsedSet.add(globalDoc);
- }
- } else {
- if (globalDoc > values.docID()) {
- values.advance(globalDoc);
- }
- if (globalDoc == values.docID()) {
- int ord = values.ordValue();
- ++count;
- BytesRef ref = values.lookupOrd(ord);
- ordBytes.put(ord, BytesRef.deepCopyOf(ref));
- groupBits.set(ord);
- collapsedSet.add(globalDoc);
- }
- }
- }
-
- if(count > 0 && count < 200) {
- try {
- groupQuery = getGroupQuery(field, count, ordBytes);
- } catch(Exception e) {
- throw new IOException(e);
- }
- }
- } else {
- groupSet = new LongHashSet(docList.size());
- NumericDocValues collapseValues = contexts.get(currentContext).reader().getNumericDocValues(field);
- int count = 0;
- for(int i=0; i<globalDocs.length; i++) {
- int globalDoc = globalDocs[i];
- while(globalDoc >= nextDocBase) {
- currentContext++;
- currentDocBase = contexts.get(currentContext).docBase;
- nextDocBase = currentContext+1 < contexts.size() ? contexts.get(currentContext+1).docBase : Integer.MAX_VALUE;
- collapseValues = contexts.get(currentContext).reader().getNumericDocValues(field);
- }
- int contextDoc = globalDoc - currentDocBase;
- int valueDocID = collapseValues.docID();
- if (valueDocID < contextDoc) {
- valueDocID = collapseValues.advance(contextDoc);
- }
- long value;
- if (valueDocID == contextDoc) {
- value = collapseValues.longValue();
- } else {
- value = 0;
- }
- if(value != nullValue) {
- ++count;
- groupSet.add(value);
- collapsedSet.add(globalDoc);
- }
- }
-
- if(count > 0 && count < 200) {
- if (fieldType.isPointField()) {
- groupQuery = getPointGroupQuery(schemaField, count, groupSet);
- } else {
- groupQuery = getGroupQuery(field, fieldType, count, groupSet);
- }
- }
- }
-
- Collector collector;
- if (sort != null)
- sort = sort.rewrite(searcher);
-
-
- Collector groupExpandCollector = null;
-
- if(values != null) {
- //Get The Top Level SortedDocValues again so we can re-iterate:
- if(CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
- @SuppressWarnings("resource") LeafReader uninvertingReader = UninvertingReader.wrap(
- new ReaderWrapper(searcher.getSlowAtomicReader(), field),
- Collections.singletonMap(field, UninvertingReader.Type.SORTED)::get);
- values = uninvertingReader.getSortedDocValues(field);
- } else {
- values = DocValues.getSorted(reader, field);
- }
-
- groupExpandCollector = new GroupExpandCollector(values, groupBits, collapsedSet, limit, sort);
- } else {
- groupExpandCollector = new NumericGroupExpandCollector(field, nullValue, groupSet, collapsedSet, limit, sort);
- }
-
- if(groupQuery != null) {
- //Limits the results to documents that are in the same group as the documents in the page.
- newFilters.add(groupQuery);
- }
-
- SolrIndexSearcher.ProcessedFilter pfilter = searcher.getProcessedFilter(null, newFilters);
- if (pfilter.postFilter != null) {
- pfilter.postFilter.setLastDelegate(groupExpandCollector);
- collector = pfilter.postFilter;
- } else {
- collector = groupExpandCollector;
- }
-
- if (pfilter.filter == null) {
- searcher.search(query, collector);
- } else {
- Query q = new BooleanQuery.Builder()
- .add(query, Occur.MUST)
- .add(pfilter.filter, Occur.FILTER)
- .build();
- searcher.search(q, collector);
- }
- LongObjectMap<Collector> groups = ((GroupCollector) groupExpandCollector).getGroups();
- NamedList outMap = new SimpleOrderedMap();
- CharsRefBuilder charsRef = new CharsRefBuilder();
- for (LongObjectCursor<Collector> cursor : groups) {
- long groupValue = cursor.key;
- TopDocsCollector<?> topDocsCollector = TopDocsCollector.class.cast(cursor.value);
- TopDocs topDocs = topDocsCollector.topDocs();
- ScoreDoc[] scoreDocs = topDocs.scoreDocs;
- if (scoreDocs.length > 0) {
- int[] docs = new int[scoreDocs.length];
- float[] scores = new float[scoreDocs.length];
- for (int i = 0; i < docs.length; i++) {
- ScoreDoc scoreDoc = scoreDocs[i];
- docs[i] = scoreDoc.doc;
- scores[i] = scoreDoc.score;
- }
- assert topDocs.totalHits.relation == TotalHits.Relation.EQUAL_TO;
- DocSlice slice = new DocSlice(0, docs.length, docs, scores, topDocs.totalHits.value, Float.NaN);
-
- if(fieldType instanceof StrField) {
- final BytesRef bytesRef = ordBytes.get((int)groupValue);
- fieldType.indexedToReadable(bytesRef, charsRef);
- String group = charsRef.toString();
- outMap.add(group, slice);
- } else {
- outMap.add(numericToString(fieldType, groupValue), slice);
- }
- }
- }
-
- rb.rsp.add("expanded", outMap);
- }
-
- @Override
- public int distributedProcess(ResponseBuilder rb) throws IOException {
- if (rb.doExpand && rb.stage < finishingStage) {
- return finishingStage;
- }
- return ResponseBuilder.STAGE_DONE;
- }
-
- @Override
- public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
- SolrParams params = rb.req.getParams();
- if (!params.getBool(COMPONENT_NAME, false)) return;
- if (!rb.onePassDistributedQuery && (sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) == 0) {
- sreq.params.set(COMPONENT_NAME, "false");
- } else {
- sreq.params.set(COMPONENT_NAME, "true");
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
-
- if (!rb.doExpand) {
- return;
- }
- if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
- SolrQueryRequest req = rb.req;
- NamedList expanded = (NamedList) req.getContext().get("expanded");
- if (expanded == null) {
- expanded = new SimpleOrderedMap();
- req.getContext().put("expanded", expanded);
- }
-
- for (ShardResponse srsp : sreq.responses) {
- NamedList response = srsp.getSolrResponse().getResponse();
- NamedList ex = (NamedList) response.get("expanded");
- for (int i=0; i<ex.size(); i++) {
- String name = ex.getName(i);
- SolrDocumentList val = (SolrDocumentList) ex.getVal(i);
- expanded.add(name, val);
- }
- }
- }
- }
-
- @Override
- public void finishStage(ResponseBuilder rb) {
-
- if (!rb.doExpand) {
- return;
- }
-
- if (rb.stage != finishingStage) {
- return;
- }
-
- NamedList expanded = (NamedList) rb.req.getContext().get("expanded");
- if (expanded == null) {
- expanded = new SimpleOrderedMap();
- }
-
- rb.rsp.add("expanded", expanded);
- }
-
- private static class GroupExpandCollector implements Collector, GroupCollector {
- private SortedDocValues docValues;
- private OrdinalMap ordinalMap;
- private SortedDocValues segmentValues;
- private LongValues segmentOrdinalMap;
- private MultiDocValues.MultiSortedDocValues multiSortedDocValues;
-
- private LongObjectMap<Collector> groups;
- private FixedBitSet groupBits;
- private IntHashSet collapsedSet;
-
- public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntHashSet collapsedSet, int limit, Sort sort) throws IOException {
- int numGroups = collapsedSet.size();
- groups = new LongObjectHashMap<>(numGroups);
- DocIdSetIterator iterator = new BitSetIterator(groupBits, 0); // cost is not useful here
- int group;
- while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- Collector collector = (sort == null) ? TopScoreDocCollector.create(limit, Integer.MAX_VALUE) : TopFieldCollector.create(sort, limit, Integer.MAX_VALUE);
- groups.put(group, collector);
- }
-
- this.collapsedSet = collapsedSet;
- this.groupBits = groupBits;
- this.docValues = docValues;
- if(docValues instanceof MultiDocValues.MultiSortedDocValues) {
- this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)docValues;
- this.ordinalMap = multiSortedDocValues.mapping;
- }
- }
-
- public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
- final int docBase = context.docBase;
-
- if(ordinalMap != null) {
- this.segmentValues = this.multiSortedDocValues.values[context.ord];
- this.segmentOrdinalMap = ordinalMap.getGlobalOrds(context.ord);
- }
-
- final LongObjectMap<LeafCollector> leafCollectors = new LongObjectHashMap<>();
- for (LongObjectCursor<Collector> entry : groups) {
- leafCollectors.put(entry.key, entry.value.getLeafCollector(context));
- }
- return new LeafCollector() {
-
- @Override
- public void setScorer(Scorable scorer) throws IOException {
- for (ObjectCursor<LeafCollector> c : leafCollectors.values()) {
- c.value.setScorer(scorer);
- }
- }
-
- @Override
- public void collect(int docId) throws IOException {
- int globalDoc = docId + docBase;
- int ord = -1;
- if(ordinalMap != null) {
- if (docId > segmentValues.docID()) {
- segmentValues.advance(docId);
- }
- if (docId == segmentValues.docID()) {
- ord = (int)segmentOrdinalMap.get(segmentValues.ordValue());
- } else {
- ord = -1;
- }
- } else {
- if (docValues.advanceExact(globalDoc)) {
- ord = docValues.ordValue();
- } else {
- ord = -1;
- }
- }
-
- if (ord > -1 && groupBits.get(ord) && !collapsedSet.contains(globalDoc)) {
- LeafCollector c = leafCollectors.get(ord);
- c.collect(docId);
- }
- }
- };
- }
-
- public LongObjectMap<Collector> getGroups() {
- return groups;
- }
- }
-
- private static class NumericGroupExpandCollector implements Collector, GroupCollector {
- private NumericDocValues docValues;
-
- private String field;
- private LongObjectHashMap<Collector> groups;
-
- private IntHashSet collapsedSet;
- private long nullValue;
-
- public NumericGroupExpandCollector(String field, long nullValue, LongHashSet groupSet, IntHashSet collapsedSet, int limit, Sort sort) throws IOException {
- int numGroups = collapsedSet.size();
- this.nullValue = nullValue;
- groups = new LongObjectHashMap<>(numGroups);
- Iterator<LongCursor> iterator = groupSet.iterator();
- while (iterator.hasNext()) {
- LongCursor cursor = iterator.next();
- Collector collector = (sort == null) ? TopScoreDocCollector.create(limit, Integer.MAX_VALUE) : TopFieldCollector.create(sort, limit, Integer.MAX_VALUE);
- groups.put(cursor.value, collector);
- }
-
- this.field = field;
- this.collapsedSet = collapsedSet;
- }
-
- public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
- final int docBase = context.docBase;
- this.docValues = context.reader().getNumericDocValues(this.field);
-
- final LongObjectHashMap<LeafCollector> leafCollectors = new LongObjectHashMap<>();
-
- for (LongObjectCursor<Collector> entry : groups) {
- leafCollectors.put(entry.key, entry.value.getLeafCollector(context));
- }
-
- return new LeafCollector() {
-
- @Override
- public void setScorer(Scorable scorer) throws IOException {
- for (ObjectCursor<LeafCollector> c : leafCollectors.values()) {
- c.value.setScorer(scorer);
- }
- }
-
- @Override
- public void collect(int docId) throws IOException {
- long value;
- if (docValues.advanceExact(docId)) {
- value = docValues.longValue();
- } else {
- value = 0;
- }
- final int index;
- if (value != nullValue &&
- (index = leafCollectors.indexOf(value)) >= 0 &&
- !collapsedSet.contains(docId + docBase)) {
- leafCollectors.indexGet(index).collect(docId);
- }
- }
- };
- }
-
- public LongObjectHashMap<Collector> getGroups() {
- return groups;
- }
-
- }
-
- //TODO lets just do simple abstract base class -- a fine use of inheritance
- private interface GroupCollector extends Collector {
- public LongObjectMap<Collector> getGroups();
-
- @Override
- default ScoreMode scoreMode() {
- final LongObjectMap<Collector> groups = getGroups();
- if (groups.isEmpty()) {
- return ScoreMode.COMPLETE; // doesn't matter?
- } else {
- return groups.iterator().next().value.scoreMode(); // we assume all the collectors should have the same nature
- }
- }
- }
-
- private Query getGroupQuery(String fname,
- FieldType ft,
- int size,
- LongHashSet groupSet) {
-
- BytesRef[] bytesRefs = new BytesRef[size];
- BytesRefBuilder term = new BytesRefBuilder();
- Iterator<LongCursor> it = groupSet.iterator();
- int index = -1;
-
- while (it.hasNext()) {
- LongCursor cursor = it.next();
- String stringVal = numericToString(ft, cursor.value);
- ft.readableToIndexed(stringVal, term);
- bytesRefs[++index] = term.toBytesRef();
- }
-
- return new TermInSetQuery(fname, bytesRefs);
- }
-
- private Query getPointGroupQuery(SchemaField sf,
- int size,
- LongHashSet groupSet) {
-
- Iterator<LongCursor> it = groupSet.iterator();
- List<String> values = new ArrayList<>(size);
- FieldType ft = sf.getType();
- while (it.hasNext()) {
- LongCursor cursor = it.next();
- values.add(numericToString(ft, cursor.value));
- }
-
- return sf.getType().getSetQuery(null, sf, values);
- }
-
- private String numericToString(FieldType fieldType, long val) {
- if (fieldType.getNumberType() != null) {
- switch (fieldType.getNumberType()) {
- case INTEGER:
- case LONG:
- return Long.toString(val);
- case FLOAT:
- return Float.toString(Float.intBitsToFloat((int)val));
- case DOUBLE:
- return Double.toString(Double.longBitsToDouble(val));
- case DATE:
- break;
- }
- }
- throw new IllegalArgumentException("FieldType must be INT,LONG,FLOAT,DOUBLE found " + fieldType);
- }
-
- private Query getGroupQuery(String fname,
- int size,
- IntObjectHashMap<BytesRef> ordBytes) throws Exception {
- BytesRef[] bytesRefs = new BytesRef[size];
- int index = -1;
- Iterator<IntObjectCursor<BytesRef>>it = ordBytes.iterator();
- while (it.hasNext()) {
- IntObjectCursor<BytesRef> cursor = it.next();
- bytesRefs[++index] = cursor.value;
- }
- return new TermInSetQuery(fname, bytesRefs);
- }
-
-
- ////////////////////////////////////////////
- /// SolrInfoBean
- ////////////////////////////////////////////
-
- @Override
- public String getDescription() {
- return "Expand Component";
- }
-
- @Override
- public Category getCategory() {
- return Category.QUERY;
- }
-
- // this reader alters the content of the given reader so it should not
- // delegate the caching stuff
- private static class ReaderWrapper extends FilterLeafReader {
-
- private String field;
-
- public ReaderWrapper(LeafReader leafReader, String field) {
- super(leafReader);
- this.field = field;
- }
-
- public SortedDocValues getSortedDocValues(String field) {
- return null;
- }
-
- public FieldInfos getFieldInfos() {
- Iterator<FieldInfo> it = in.getFieldInfos().iterator();
- List<FieldInfo> newInfos = new ArrayList<>();
- while(it.hasNext()) {
- FieldInfo fieldInfo = it.next();
-
- if(fieldInfo.name.equals(field)) {
- FieldInfo f = new FieldInfo(fieldInfo.name,
- fieldInfo.number,
- fieldInfo.hasVectors(),
- fieldInfo.hasNorms(),
- fieldInfo.hasPayloads(),
- fieldInfo.getIndexOptions(),
- DocValuesType.NONE,
- fieldInfo.getDocValuesGen(),
- fieldInfo.attributes(),
- fieldInfo.getPointDataDimensionCount(),
- fieldInfo.getPointIndexDimensionCount(),
- fieldInfo.getPointNumBytes(),
- fieldInfo.isSoftDeletesField());
- newInfos.add(f);
-
- } else {
- newInfos.add(fieldInfo);
- }
- }
- FieldInfos infos = new FieldInfos(newInfos.toArray(new FieldInfo[newInfos.size()]));
- return infos;
- }
-
- // NOTE: delegating the caches is wrong here as we are altering the content
- // of the reader, this should ONLY be used under an uninvertingreader which
- // will restore doc values back using uninversion, otherwise all sorts of
- // crazy things could happen.
-
- @Override
- public CacheHelper getCoreCacheHelper() {
- return in.getCoreCacheHelper();
- }
-
- @Override
- public CacheHelper getReaderCacheHelper() {
- return in.getReaderCacheHelper();
- }
- }
-
-}