You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ep...@apache.org on 2023/07/31 16:46:23 UTC
[solr] branch branch_9x updated (49419a562a1 -> a6e6730fba0)
This is an automated email from the ASF dual-hosted git repository.
epugh pushed a change to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
from 49419a562a1 SOLR-16490: Create v2 equivalent for core backup API (#1740)
new 870b1d8a228 SOLR-16883: Use postlogs tool from windows and unix via solr cli infrastructure. (#1786)
new a6e6730fba0 complete backport of SOLR-16883
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
solr/CHANGES.txt | 4 +-
solr/bin/postlogs | 1 +
solr/bin/solr | 4 +-
solr/bin/solr.cmd | 4 +-
.../{SolrLogPostTool.java => PostLogsTool.java} | 69 ++-
.../core/src/java/org/apache/solr/cli/SolrCLI.java | 1 +
.../java/org/apache/solr/cli/SolrLogPostTool.java | 549 +--------------------
...rLogPostToolTest.java => PostLogsToolTest.java} | 7 +-
solr/packaging/test/test_postlogs.bats | 14 +-
.../modules/query-guide/pages/logs.adoc | 40 +-
10 files changed, 108 insertions(+), 585 deletions(-)
copy solr/core/src/java/org/apache/solr/cli/{SolrLogPostTool.java => PostLogsTool.java} (93%)
rename solr/core/src/test/org/apache/solr/cli/{SolrLogPostToolTest.java => PostLogsToolTest.java} (98%)
[solr] 01/02: SOLR-16883: Use postlogs tool from windows and unix via solr cli infrastructure. (#1786)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 870b1d8a22813d313079bc2497e4c64529f8914f
Author: Eric Pugh <ep...@opensourceconnections.com>
AuthorDate: Mon Jul 31 07:43:33 2023 -0400
SOLR-16883: Use postlogs tool from windows and unix via solr cli infrastructure. (#1786)
Postlogs tool only works on Linux, and was it's own implementation of a command line tool, with different patterns then the other tools. Migrate over to "bin/solr postlogs" and it now works like other CLI tools, including working on Windows.
---
solr/CHANGES.txt | 4 +-
solr/bin/postlogs | 1 +
solr/bin/solr | 2 +-
.../{SolrLogPostTool.java => PostLogsTool.java} | 69 ++-
.../core/src/java/org/apache/solr/cli/SolrCLI.java | 1 +
.../java/org/apache/solr/cli/SolrLogPostTool.java | 549 +--------------------
...rLogPostToolTest.java => PostLogsToolTest.java} | 7 +-
solr/packaging/test/test_postlogs.bats | 14 +-
.../modules/query-guide/pages/logs.adoc | 40 +-
9 files changed, 104 insertions(+), 583 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a80e887b9b6..c7889a598cb 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -15,6 +15,8 @@ Improvements
* SOLR-16490: `/admin/cores?action=backupcore` now has a v2 equivalent, available at
`GET /api/cores/coreName/backups` (Sanjay Dutt via Jason Gerlowski)
+* SOLR-16883: Postlogs tool for indexing Solr logs in Solr now supported on Windows by converting it to a Solr CLI command: `bin/solr postlogs`. `bin/postlogs` script marked deprected. (Eric Pugh, Will White)
+
Optimizations
---------------------
@@ -179,7 +181,7 @@ Improvements
* SOLR-16687: Add support of SolrClassLoader to SolrZkClient (Lamine Idjeraoui via Jason Gerlowski & Houston Putman)
-* SOLR-9378: Internal shard requests no longer include the wasteful shard.url param. [shard] transformer now defaults to returning
+* SOLR-9378: Internal shard requests no longer include the wasteful shard.url param. [shard] transformer now defaults to returning
only the shard id (based on luceneMatchVersion), but can be configured to return the legacy list of replicas. (hossman)
* SOLR-16816: Update node metrics while making affinityPlacement selections. Therefore selections can be made given the expected cluster
diff --git a/solr/bin/postlogs b/solr/bin/postlogs
index 71a5e7a255d..249f693230d 100755
--- a/solr/bin/postlogs
+++ b/solr/bin/postlogs
@@ -31,6 +31,7 @@
#
############################################################################################
+echo "This script has been deprecated in favour of 'bin/solr postlogs' command."
SOLR_TIP="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/..
java -classpath "$SOLR_TIP/server/lib/ext/*:$SOLR_TIP/server/solr-webapp/webapp/WEB-INF/lib/*" org.apache.solr.cli.SolrLogPostTool $1 $2
diff --git a/solr/bin/solr b/solr/bin/solr
index 9183d840db0..44408103603 100644
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -940,7 +940,7 @@ if [ $# -eq 1 ]; then
-help|-h)
print_usage ""
exit
- ;;
+ ;;
esac
fi
diff --git a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java
similarity index 93%
copy from solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java
copy to solr/core/src/java/org/apache/solr/cli/PostLogsTool.java
index 8ee87a3b8a8..91e1a4e6fbf 100644
--- a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java
@@ -14,11 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.solr.cli;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.LineNumberReader;
+import java.io.PrintStream;
import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.nio.file.Files;
@@ -32,6 +34,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.request.UpdateRequest;
@@ -40,29 +44,48 @@ import org.apache.solr.common.SolrInputField;
import org.apache.solr.handler.component.ShardRequest;
/** A command line tool for indexing Solr logs in the out-of-the-box log format. */
-public class SolrLogPostTool {
-
- public static void main(String[] args) throws Exception {
-
- if (args.length != 2) {
- CLIO.out("");
- CLIO.out("postlogs is a simple tool for indexing Solr logs.");
- CLIO.out("");
- CLIO.out("parameters:");
- CLIO.out("");
- CLIO.out("-- baseUrl: Example http://localhost:8983/solr/collection1");
- CLIO.out("-- rootDir: All files found at or below the root will be indexed.");
- CLIO.out("");
- CLIO.out(
- "Sample syntax 1: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs/solr.log");
- CLIO.out(
- "Sample syntax 2: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs");
- CLIO.out("");
- return;
- }
+public class PostLogsTool extends ToolBase {
+
+ public PostLogsTool() {
+ this(CLIO.getOutStream());
+ }
+
+ public PostLogsTool(PrintStream stdout) {
+ super(stdout);
+ }
+
+ @Override
+ public String getName() {
+ return "postlogs";
+ }
+
+ @Override
+ public List<Option> getOptions() {
+ return List.of(
+ Option.builder("url")
+ .longOpt("url")
+ .argName("ADDRESS")
+ .hasArg()
+ .required(true)
+ .desc("Address of the collection, example http://localhost:8983/solr/collection1/.")
+ .build(),
+ Option.builder("rootdir")
+ .longOpt("rootdir")
+ .argName("DIRECTORY")
+ .hasArg()
+ .required(true)
+ .desc("All files found at or below the root directory will be indexed.")
+ .build());
+ }
+
+ @Override
+ public void runImpl(CommandLine cli) throws Exception {
+ String url = cli.getOptionValue("url");
+ String rootDir = cli.getOptionValue("rootdir");
+ runCommand(url, rootDir);
+ }
- String baseUrl = args[0];
- String root = args[1];
+ public void runCommand(String baseUrl, String root) throws IOException {
Http2SolrClient.Builder builder = new Http2SolrClient.Builder(baseUrl);
try (SolrClient client = builder.build()) {
@@ -114,7 +137,7 @@ public class SolrLogPostTool {
}
}
- private static void sendBatch(SolrClient client, UpdateRequest request, boolean lastRequest) {
+ private void sendBatch(SolrClient client, UpdateRequest request, boolean lastRequest) {
final String beginMessage =
lastRequest ? "Sending last batch ..." : "Sending batch of 300 log records...";
CLIO.out(beginMessage);
diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
index b0288f2f870..ce1ea16d564 100755
--- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
@@ -240,6 +240,7 @@ public class SolrCLI implements CLIO {
else if ("auth".equals(toolType)) return new AuthTool();
else if ("export".equals(toolType)) return new ExportTool();
else if ("package".equals(toolType)) return new PackageTool();
+ else if ("postlogs".equals(toolType)) return new PostLogsTool();
else if ("version".equals(toolType)) return new VersionTool();
// If you add a built-in tool to this class, add it here to avoid
diff --git a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java b/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java
index 8ee87a3b8a8..cf4fee63b35 100644
--- a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java
@@ -16,30 +16,12 @@
*/
package org.apache.solr.cli;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.LineNumberReader;
-import java.net.URLDecoder;
-import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.UUID;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.impl.Http2SolrClient;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.SolrInputField;
-import org.apache.solr.handler.component.ShardRequest;
-
-/** A command line tool for indexing Solr logs in the out-of-the-box log format. */
+/**
+ * A command line tool for indexing Solr logs in the out-of-the-box log format.
+ *
+ * @deprecated Please use {@link PostLogsTool} that is exposed as 'bin/solr postlogs'.
+ */
+@Deprecated(since = "9.4")
public class SolrLogPostTool {
public static void main(String[] args) throws Exception {
@@ -60,524 +42,9 @@ public class SolrLogPostTool {
CLIO.out("");
return;
}
-
String baseUrl = args[0];
String root = args[1];
-
- Http2SolrClient.Builder builder = new Http2SolrClient.Builder(baseUrl);
- try (SolrClient client = builder.build()) {
- int rec = 0;
- UpdateRequest request = new UpdateRequest();
-
- List<Path> files;
- try (Stream<Path> stream = Files.walk(Path.of(root), Integer.MAX_VALUE)) {
- files = stream.filter(Files::isRegularFile).collect(Collectors.toList());
- }
-
- for (Path file : files) {
- try (LineNumberReader reader =
- new LineNumberReader(Files.newBufferedReader(file, Charset.defaultCharset()))) {
- LogRecordReader recordReader = new LogRecordReader(reader);
- SolrInputDocument doc;
- String fileName = file.getFileName().toString();
- while (true) {
- try {
- doc = recordReader.readRecord();
- } catch (Throwable t) {
- CLIO.err(
- "Error reading log record:" + reader.getLineNumber() + " from file:" + fileName);
- CLIO.err(t.getMessage());
- continue;
- }
-
- if (doc == null) {
- break;
- }
-
- rec++;
- UUID id = UUID.randomUUID();
- doc.setField("id", id.toString());
- doc.setField("file_s", fileName);
- request.add(doc);
- if (rec == 300) {
- sendBatch(client, request, false /* normal batch */);
- request = new UpdateRequest();
- rec = 0;
- }
- }
- }
- }
-
- if (rec > 0) {
- sendBatch(client, request, true /* last batch */);
- }
- }
- }
-
- private static void sendBatch(SolrClient client, UpdateRequest request, boolean lastRequest) {
- final String beginMessage =
- lastRequest ? "Sending last batch ..." : "Sending batch of 300 log records...";
- CLIO.out(beginMessage);
- try {
- request.process(client);
- CLIO.out("Batch sent");
- } catch (Exception e) {
- CLIO.err("Batch sending failed: " + e.getMessage());
- e.printStackTrace(CLIO.getErrStream());
- }
-
- if (lastRequest) {
- try {
- client.commit();
- CLIO.out("Committed");
- } catch (Exception e) {
- CLIO.err("Unable to commit documents: " + e.getMessage());
- e.printStackTrace(CLIO.getErrStream());
- }
- }
- }
-
- public static class LogRecordReader {
-
- private BufferedReader bufferedReader;
- private String pushedBack = null;
- private boolean finished = false;
- private String cause;
- private Pattern p =
- Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d[\\s|T]\\d\\d:\\d\\d\\:\\d\\d.\\d\\d\\d)");
- private Pattern minute =
- Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d[\\s|T]\\d\\d:\\d\\d)");
- private Pattern tenSecond =
- Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d[\\s|T]\\d\\d:\\d\\d:\\d)");
-
- public LogRecordReader(BufferedReader bufferedReader) throws IOException {
- this.bufferedReader = bufferedReader;
- }
-
- public SolrInputDocument readRecord() throws IOException {
- while (true) {
- String line = null;
-
- if (finished) {
- return null;
- }
-
- if (pushedBack != null) {
- line = pushedBack;
- pushedBack = null;
- } else {
- line = bufferedReader.readLine();
- }
-
- if (line != null) {
- SolrInputDocument lineDoc = new SolrInputDocument();
- String date = parseDate(line);
- String minute = parseMinute(line);
- String tenSecond = parseTenSecond(line);
- lineDoc.setField("date_dt", date);
- lineDoc.setField("time_minute_s", minute);
- lineDoc.setField("time_ten_second_s", tenSecond);
- lineDoc.setField("line_t", line);
- lineDoc.setField("type_s", "other"); // Overridden by known types below
-
- if (line.contains("Registered new searcher")) {
- parseNewSearch(lineDoc, line);
- } else if (line.contains("path=/update")) {
- parseUpdate(lineDoc, line);
- } else if (line.contains(" ERROR ")) {
- this.cause = null;
- parseError(lineDoc, line, readTrace());
- } else if (line.contains("QTime=")) {
- parseQueryRecord(lineDoc, line);
- }
-
- return lineDoc;
- } else {
- return null;
- }
- }
- }
-
- private String readTrace() throws IOException {
- StringBuilder buf = new StringBuilder();
- buf.append("%html ");
-
- while (true) {
- String line = bufferedReader.readLine();
- if (line == null) {
- finished = true;
- return buf.toString();
- } else {
- // look for a date at the beginning of the line
- // If it's not there then read into the stack trace buffer
- Matcher m = p.matcher(line);
-
- if (!m.find() && buf.length() < 10000) {
- // Line does not start with a timestamp so append to the stack trace
- buf.append(line.replace("\t", " ")).append("<br/>");
- if (line.startsWith("Caused by:")) {
- this.cause = line;
- }
- } else {
- pushedBack = line;
- break;
- }
- }
- }
-
- return buf.toString();
- }
-
- private String parseDate(String line) {
- Matcher m = p.matcher(line);
- if (m.find()) {
- String date = m.group(1);
- return date.replace(" ", "T") + "Z";
- }
-
- return null;
- }
-
- private String parseMinute(String line) {
- Matcher m = minute.matcher(line);
- if (m.find()) {
- String date = m.group(1);
- return date.replace(" ", "T") + ":00Z";
- }
-
- return null;
- }
-
- private String parseTenSecond(String line) {
- Matcher m = tenSecond.matcher(line);
- if (m.find()) {
- String date = m.group(1);
- return date.replace(" ", "T") + "0Z";
- }
-
- return null;
- }
-
- private void setFieldIfUnset(SolrInputDocument doc, String fieldName, String fieldValue) {
- if (doc.containsKey(fieldName)) return;
-
- doc.setField(fieldName, fieldValue);
- }
-
- private void parseError(SolrInputDocument lineRecord, String line, String trace) {
- lineRecord.setField("type_s", "error");
-
- // Don't include traces that have only the %html header.
- if (trace != null && trace.length() > 6) {
- lineRecord.setField("stack_t", trace);
- }
-
- if (this.cause != null) {
- lineRecord.setField("root_cause_t", cause.replace("Caused by:", "").trim());
- }
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
- }
-
- private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("qtime_i", parseQTime(line));
- lineRecord.setField("status_s", parseStatus(line));
-
- String path = parsePath(line);
- lineRecord.setField("path_s", path);
-
- if (line.contains("hits=")) {
- lineRecord.setField("hits_l", parseHits(line));
- }
-
- String params = parseParams(line);
- lineRecord.setField("params_t", params);
- addParams(lineRecord, params);
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("node_s", parseNode(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
-
- if (path != null && path.contains("/admin")) {
- lineRecord.setField("type_s", "admin");
- } else if (path != null && params.contains("/replication")) {
- lineRecord.setField("type_s", "replication");
- } else if (path != null && path.contains("/get")) {
- lineRecord.setField("type_s", "get");
- } else {
- lineRecord.setField("type_s", "query");
- }
- }
-
- private void parseNewSearch(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("type_s", "newSearcher");
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
- }
-
- private String parseCollection(String line) {
- char[] ca = {' ', ']', ','};
- String[] parts = line.split("c:");
- if (parts.length >= 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private void parseUpdate(SolrInputDocument lineRecord, String line) {
- if (line.contains("deleteByQuery=")) {
- lineRecord.setField("type_s", "deleteByQuery");
- } else if (line.contains("delete=")) {
- lineRecord.setField("type_s", "delete");
- } else if (line.contains("commit=true")) {
- lineRecord.setField("type_s", "commit");
- } else {
- lineRecord.setField("type_s", "update");
- }
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
- }
-
- private String parseCore(String line) {
- char[] ca = {' ', ']', '}', ','};
- String[] parts = line.split("x:");
- if (parts.length >= 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseShard(String line) {
- char[] ca = {' ', ']', '}', ','};
- String[] parts = line.split("s:");
- if (parts.length >= 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseReplica(String line) {
- char[] ca = {' ', ']', '}', ','};
- String[] parts = line.split("r:");
- if (parts.length >= 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parsePath(String line) {
- char[] ca = {' '};
- String[] parts = line.split(" path=");
- if (parts.length == 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseQTime(String line) {
- char[] ca = {'\n', '\r'};
- String[] parts = line.split(" QTime=");
- if (parts.length == 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseNode(String line) {
- char[] ca = {' ', ']', '}', ','};
- String[] parts = line.split("node_name=n:");
- if (parts.length >= 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseStatus(String line) {
- char[] ca = {' ', '\n', '\r'};
- String[] parts = line.split(" status=");
- if (parts.length == 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseHits(String line) {
- char[] ca = {' '};
- String[] parts = line.split(" hits=");
- if (parts.length == 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseParams(String line) {
- char[] ca = {' '};
- String[] parts = line.split(" params=");
- if (parts.length == 2) {
- String p = readUntil(parts[1].substring(1), ca);
- return p.substring(0, p.length() - 1);
- } else {
- return null;
- }
- }
-
- private String readUntil(String s, char[] chars) {
- StringBuilder builder = new StringBuilder();
- for (int i = 0; i < s.length(); i++) {
- char a = s.charAt(i);
- for (char c : chars) {
- if (a == c) {
- return builder.toString();
- }
- }
- builder.append(a);
- }
-
- return builder.toString();
- }
-
- private void addParams(SolrInputDocument doc, String params) {
- String[] pairs = params.split("&");
- for (String pair : pairs) {
- String[] parts = pair.split("=");
- if (parts.length == 2 && parts[0].equals("q")) {
- String dq = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "q_s", dq);
- setFieldIfUnset(doc, "q_t", dq);
- }
-
- if (parts[0].equals("rows")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "rows_i", dr);
- }
-
- if (parts[0].equals("start")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "start_i", dr);
- }
-
- if (parts[0].equals("distrib")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "distrib_s", dr);
- }
-
- if (parts[0].equals("shards")) {
- setFieldIfUnset(doc, "shards_s", "true");
- }
-
- if (parts[0].equals("ids") && !isRTGRequest(doc)) {
- setFieldIfUnset(doc, "ids_s", "true");
- }
-
- if (parts[0].equals("isShard")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "isShard_s", dr);
- }
-
- if (parts[0].equals("wt")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "wt_s", dr);
- }
-
- if (parts[0].equals("facet")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "facet_s", dr);
- }
-
- if (parts[0].equals("shards.purpose")) {
- try {
- int purpose = Integer.parseInt(parts[1]);
- String[] purposes = getRequestPurposeNames(purpose);
- for (String p : purposes) {
- doc.addField("purpose_ss", p);
- }
- } catch (Throwable e) {
- // We'll just sit on this for now and not interrupt the load for this one field.
- }
- }
- }
-
- // Special params used to determine what stage a query is.
- // So we populate with defaults.
- // The absence of the distrib params means it's a distributed query.
- setFieldIfUnset(doc, "distrib_s", "true");
- setFieldIfUnset(doc, "shards_s", "false");
- setFieldIfUnset(doc, "ids_s", "false");
- }
-
- private boolean isRTGRequest(SolrInputDocument doc) {
- final SolrInputField path = doc.getField("path_s");
- if (path == null) return false;
-
- return "/get".equals(path.getValue());
- }
- }
-
- private static final Map<Integer, String> purposes;
- protected static final String UNKNOWN_VALUE = "Unknown";
- private static final String[] purposeUnknown = new String[] {UNKNOWN_VALUE};
-
- public static String[] getRequestPurposeNames(Integer reqPurpose) {
- if (reqPurpose != null) {
- int valid = 0;
- for (Map.Entry<Integer, String> entry : purposes.entrySet()) {
- if ((reqPurpose & entry.getKey()) != 0) {
- valid++;
- }
- }
- if (valid == 0) {
- return purposeUnknown;
- } else {
- String[] result = new String[valid];
- int i = 0;
- for (Map.Entry<Integer, String> entry : purposes.entrySet()) {
- if ((reqPurpose & entry.getKey()) != 0) {
- result[i] = entry.getValue();
- i++;
- }
- }
- return result;
- }
- }
- return purposeUnknown;
- }
-
- static {
- Map<Integer, String> map = new TreeMap<>();
- map.put(ShardRequest.PURPOSE_PRIVATE, "PRIVATE");
- map.put(ShardRequest.PURPOSE_GET_TOP_IDS, "GET_TOP_IDS");
- map.put(ShardRequest.PURPOSE_REFINE_TOP_IDS, "REFINE_TOP_IDS");
- map.put(ShardRequest.PURPOSE_GET_FACETS, "GET_FACETS");
- map.put(ShardRequest.PURPOSE_REFINE_FACETS, "REFINE_FACETS");
- map.put(ShardRequest.PURPOSE_GET_FIELDS, "GET_FIELDS");
- map.put(ShardRequest.PURPOSE_GET_HIGHLIGHTS, "GET_HIGHLIGHTS");
- map.put(ShardRequest.PURPOSE_GET_DEBUG, "GET_DEBUG");
- map.put(ShardRequest.PURPOSE_GET_STATS, "GET_STATS");
- map.put(ShardRequest.PURPOSE_GET_TERMS, "GET_TERMS");
- map.put(ShardRequest.PURPOSE_GET_TOP_GROUPS, "GET_TOP_GROUPS");
- map.put(ShardRequest.PURPOSE_GET_MLT_RESULTS, "GET_MLT_RESULTS");
- map.put(ShardRequest.PURPOSE_REFINE_PIVOT_FACETS, "REFINE_PIVOT_FACETS");
- map.put(ShardRequest.PURPOSE_SET_TERM_STATS, "SET_TERM_STATS");
- map.put(ShardRequest.PURPOSE_GET_TERM_STATS, "GET_TERM_STATS");
- purposes = Collections.unmodifiableMap(map);
+ PostLogsTool postLogsTool = new PostLogsTool();
+ postLogsTool.runCommand(baseUrl, root);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cli/SolrLogPostToolTest.java b/solr/core/src/test/org/apache/solr/cli/PostLogsToolTest.java
similarity index 98%
rename from solr/core/src/test/org/apache/solr/cli/SolrLogPostToolTest.java
rename to solr/core/src/test/org/apache/solr/cli/PostLogsToolTest.java
index 2ebfb9f1b82..6240c5443b1 100644
--- a/solr/core/src/test/org/apache/solr/cli/SolrLogPostToolTest.java
+++ b/solr/core/src/test/org/apache/solr/cli/PostLogsToolTest.java
@@ -22,20 +22,19 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.cli.SolrLogPostTool.LogRecordReader;
+import org.apache.solr.cli.PostLogsTool.LogRecordReader;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.junit.BeforeClass;
import org.junit.Test;
-public class SolrLogPostToolTest extends SolrTestCaseJ4 {
+public class PostLogsToolTest extends SolrTestCaseJ4 {
private static boolean solr9Format;
@BeforeClass
public static void beforeClass() {
solr9Format = random().nextBoolean();
- System.out.println("Solr 9 Format: " + solr9Format);
}
private String sometimesSolr9Format(String record) {
@@ -310,7 +309,7 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
assertEquals(collection.getValue(), "test");
}
- // Ensure SolrLogPostTool parses _all_ log lines into searchable records
+ // Ensure PostLogsTool parses _all_ log lines into searchable records
@Test
public void testOtherRecord() throws Exception {
final String record =
diff --git a/solr/packaging/test/test_postlogs.bats b/solr/packaging/test/test_postlogs.bats
index 32f7c379a7c..64c8230fefd 100644
--- a/solr/packaging/test/test_postlogs.bats
+++ b/solr/packaging/test/test_postlogs.bats
@@ -38,7 +38,7 @@ teardown() {
delete_all_collections
}
-@test "post solr log into solr" {
+@test "post solr log into solr via script" {
run solr create_collection -c COLL_NAME
assert_output --partial "Created collection 'COLL_NAME'"
@@ -49,3 +49,15 @@ teardown() {
run curl 'http://localhost:8983/solr/COLL_NAME/select?q=*:*'
refute_output --partial '"numFound":0'
}
+
+@test "post solr log into solr via cli" {
+ run solr create_collection -c COLL_NAME
+ assert_output --partial "Created collection 'COLL_NAME'"
+
+ run solr postlogs -url http://localhost:8983/solr/COLL_NAME -rootdir ${SOLR_LOGS_DIR}/solr.log
+ assert_output --partial 'Sending last batch'
+ assert_output --partial 'Committed'
+
+ run curl 'http://localhost:8983/solr/COLL_NAME/select?q=*:*'
+ refute_output --partial '"numFound":0'
+}
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc b/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc
index 7b9412570bd..7ac02556335 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc
@@ -24,25 +24,41 @@ See the xref:math-start.adoc[] chapter to learn how to get started with visualiz
== Loading
-The out-of-the-box Solr log format can be loaded into a Solr index using the `bin/postlogs` command line tool located in the `bin/` directory of the Solr distribution.
+The out-of-the-box Solr log format can be loaded into a Solr index using the `bin/solr postlogs` command line tool located in the `bin/` directory of the Solr distribution.
-NOTE: If working from the source distribution the
-distribution must first be built before `postlogs` can be run.
+=== Postlogs
-The `postlogs` script is designed to be run from the root directory of the Solr distribution.
+The `postlogs` command reads in Solr's log format and indexes it in a Solr collection.
-The `postlogs` script takes two parameters:
+`bin/solr postlogs [options]`
-* Solr base URL (with collection): `http://localhost:8983/solr/logs`
-* File path to root of the logs directory: All files found under this directory (including sub-directories) will be indexed.
+`bin/solr postlogs -help`
+
+==== Healthcheck Parameters
+
+`-url <ADDRESS>`::
++
+[%autowidth,frame=none]
+|===
+|Required |Default: none
+|===
++
+Address of the collection, example http://localhost:8983/solr/collection1/.
++
+
+`-rootdir <DIRECTORY>`::
++
+[%autowidth,frame=none]
+|===
+|Required |Default: none
+|===
++
+File path to root of the logs directory: All files found under this directory (including sub-directories) will be indexed.
If the path points to a single log file only that log file will be loaded.
-Below is a sample execution of the `postlogs` tool:
++
+*Example*: `bin/solr postlogs --url http://localhost:8983/solr/logs --rootdir /var/logs/solrlogs`
-[source,text]
-----
-./bin/postlogs http://localhost:8983/solr/logs /var/logs/solrlogs
-----
The example above will index all the log files under `/var/logs/solrlogs` to the `logs` collection found at the base url `http://localhost:8983/solr`.
[solr] 02/02: complete backport of SOLR-16883
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit a6e6730fba0d4e764fca8afa36ca5a4a44bbffee
Author: Eric Pugh <ep...@opensourceconnections.com>
AuthorDate: Mon Jul 31 12:44:45 2023 -0400
complete backport of SOLR-16883
---
solr/bin/solr | 2 +-
solr/bin/solr.cmd | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/solr/bin/solr b/solr/bin/solr
index 44408103603..79b1c6c8046 100644
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -342,7 +342,7 @@ function print_usage() {
if [ -z "${CMD:-}" ]; then
echo ""
echo "Usage: solr COMMAND OPTIONS"
- echo " where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version, zk, auth, assert, config, export, api, package"
+ echo " where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version, zk, auth, assert, config, export, api, package, postlogs"
echo ""
echo " Standalone server example (start Solr running in the background on port 8984):"
echo ""
diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd
index a75f882f407..bfdeab06c92 100755
--- a/solr/bin/solr.cmd
+++ b/solr/bin/solr.cmd
@@ -236,6 +236,7 @@ IF "%1"=="assert" goto run_solrcli
IF "%1"=="export" goto run_solrcli
IF "%1"=="package" goto run_solrcli
IF "%1"=="api" goto run_solrcli
+IF "%1"=="postlogs" goto run_solrcli
REM Only allow the command to be the first argument, assume start if not supplied
IF "%1"=="start" goto set_script_cmd
@@ -307,12 +308,13 @@ IF "%SCRIPT_CMD%"=="delete" goto delete_usage
IF "%SCRIPT_CMD%"=="zk" goto zk_usage
IF "%SCRIPT_CMD%"=="auth" goto auth_usage
IF "%SCRIPT_CMD%"=="status" goto status_usage
+IF "%SCRIPT_CMD%"=="postlogs" goto run_solrcli
goto done
:script_usage
@echo.
@echo Usage: solr COMMAND OPTIONS
-@echo where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version, zk, auth, assert, config, export, api, package
+@echo where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version, zk, auth, assert, config, export, api, package, postlogs
@echo.
@echo Standalone server example (start Solr running in the background on port 8984):
@echo.