You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ep...@apache.org on 2023/07/31 11:43:38 UTC
[solr] branch main updated: SOLR-16883: Use postlogs tool from windows and unix via solr cli infrastructure. (#1786)
This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 279b9cd4e93 SOLR-16883: Use postlogs tool from windows and unix via solr cli infrastructure. (#1786)
279b9cd4e93 is described below
commit 279b9cd4e93dee39c8097fef877dda98cb696fb7
Author: Eric Pugh <ep...@opensourceconnections.com>
AuthorDate: Mon Jul 31 07:43:33 2023 -0400
SOLR-16883: Use postlogs tool from windows and unix via solr cli infrastructure. (#1786)
Postlogs tool only works on Linux, and was it's own implementation of a command line tool, with different patterns then the other tools. Migrate over to "bin/solr postlogs" and it now works like other CLI tools, including working on Windows.
---
solr/CHANGES.txt | 2 +
solr/bin/postlogs | 1 +
solr/bin/solr | 8 +-
solr/bin/solr.cmd | 2 +
.../{SolrLogPostTool.java => PostLogsTool.java} | 69 ++-
.../core/src/java/org/apache/solr/cli/SolrCLI.java | 1 +
.../java/org/apache/solr/cli/SolrLogPostTool.java | 549 +--------------------
...rLogPostToolTest.java => PostLogsToolTest.java} | 7 +-
solr/packaging/test/test_postlogs.bats | 14 +-
.../modules/query-guide/pages/logs.adoc | 40 +-
10 files changed, 108 insertions(+), 585 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 2cd77f1e1a9..721d446250d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -65,6 +65,8 @@ Improvements
* SOLR-16490: `/admin/cores?action=backupcore` now has a v2 equivalent, available at
`GET /api/cores/coreName/backups` (Sanjay Dutt via Jason Gerlowski)
+* SOLR-16883: Postlogs tool for indexing Solr logs in Solr now supported on Windows by converting it to a Solr CLI command: `bin/solr postlogs`. `bin/postlogs` script marked deprected. (Eric Pugh, Will White)
+
Optimizations
---------------------
diff --git a/solr/bin/postlogs b/solr/bin/postlogs
index 71a5e7a255d..249f693230d 100755
--- a/solr/bin/postlogs
+++ b/solr/bin/postlogs
@@ -31,6 +31,7 @@
#
############################################################################################
+echo "This script has been deprecated in favour of 'bin/solr postlogs' command."
SOLR_TIP="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/..
java -classpath "$SOLR_TIP/server/lib/ext/*:$SOLR_TIP/server/solr-webapp/webapp/WEB-INF/lib/*" org.apache.solr.cli.SolrLogPostTool $1 $2
diff --git a/solr/bin/solr b/solr/bin/solr
index 09053117057..396de287593 100644
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -795,10 +795,10 @@ function stop_solr() {
if [ $# -eq 1 ]; then
case $1 in
- -help|-h)
- run_tool ""
+ -help|-h)
+ run_tool ""
exit
- ;;
+ ;;
esac
fi
@@ -861,7 +861,7 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
upconfig|downconfig|cp|rm|mv|ls|mkroot)
ZK_OP=$1
shift 1
- ;;
+ ;;
-z|-zkhost|-zkHost)
if [[ -z "$2" || "${2:0:1}" == "-" ]]; then
print_short_zk_usage "$SCRIPT_CMD" "ZooKeeper connection string is required when using the $1 option!"
diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd
index ae16c5dacb5..7936998ef20 100755
--- a/solr/bin/solr.cmd
+++ b/solr/bin/solr.cmd
@@ -247,6 +247,7 @@ IF "%1"=="create" goto run_solrcli
IF "%1"=="create_core" goto run_solrcli
IF "%1"=="create_collection" goto run_solrcli
IF "%1"=="delete" goto run_solrcli
+IF "%1"=="postlogs" goto run_solrcli
IF "%1"=="zk" (
set SCRIPT_CMD=zk
SHIFT
@@ -281,6 +282,7 @@ IF "%SCRIPT_CMD%"=="delete" goto run_solrcli
IF "%SCRIPT_CMD%"=="zk" goto zk_usage
IF "%SCRIPT_CMD%"=="auth" goto auth_usage
IF "%SCRIPT_CMD%"=="status" goto run_solrcli
+IF "%SCRIPT_CMD%"=="postlogs" goto run_solrcli
goto done
:start_usage
diff --git a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java
similarity index 93%
copy from solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java
copy to solr/core/src/java/org/apache/solr/cli/PostLogsTool.java
index 8ee87a3b8a8..91e1a4e6fbf 100644
--- a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java
@@ -14,11 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.solr.cli;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.LineNumberReader;
+import java.io.PrintStream;
import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.nio.file.Files;
@@ -32,6 +34,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.request.UpdateRequest;
@@ -40,29 +44,48 @@ import org.apache.solr.common.SolrInputField;
import org.apache.solr.handler.component.ShardRequest;
/** A command line tool for indexing Solr logs in the out-of-the-box log format. */
-public class SolrLogPostTool {
-
- public static void main(String[] args) throws Exception {
-
- if (args.length != 2) {
- CLIO.out("");
- CLIO.out("postlogs is a simple tool for indexing Solr logs.");
- CLIO.out("");
- CLIO.out("parameters:");
- CLIO.out("");
- CLIO.out("-- baseUrl: Example http://localhost:8983/solr/collection1");
- CLIO.out("-- rootDir: All files found at or below the root will be indexed.");
- CLIO.out("");
- CLIO.out(
- "Sample syntax 1: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs/solr.log");
- CLIO.out(
- "Sample syntax 2: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs");
- CLIO.out("");
- return;
- }
+public class PostLogsTool extends ToolBase {
+
+ public PostLogsTool() {
+ this(CLIO.getOutStream());
+ }
+
+ public PostLogsTool(PrintStream stdout) {
+ super(stdout);
+ }
+
+ @Override
+ public String getName() {
+ return "postlogs";
+ }
+
+ @Override
+ public List<Option> getOptions() {
+ return List.of(
+ Option.builder("url")
+ .longOpt("url")
+ .argName("ADDRESS")
+ .hasArg()
+ .required(true)
+ .desc("Address of the collection, example http://localhost:8983/solr/collection1/.")
+ .build(),
+ Option.builder("rootdir")
+ .longOpt("rootdir")
+ .argName("DIRECTORY")
+ .hasArg()
+ .required(true)
+ .desc("All files found at or below the root directory will be indexed.")
+ .build());
+ }
+
+ @Override
+ public void runImpl(CommandLine cli) throws Exception {
+ String url = cli.getOptionValue("url");
+ String rootDir = cli.getOptionValue("rootdir");
+ runCommand(url, rootDir);
+ }
- String baseUrl = args[0];
- String root = args[1];
+ public void runCommand(String baseUrl, String root) throws IOException {
Http2SolrClient.Builder builder = new Http2SolrClient.Builder(baseUrl);
try (SolrClient client = builder.build()) {
@@ -114,7 +137,7 @@ public class SolrLogPostTool {
}
}
- private static void sendBatch(SolrClient client, UpdateRequest request, boolean lastRequest) {
+ private void sendBatch(SolrClient client, UpdateRequest request, boolean lastRequest) {
final String beginMessage =
lastRequest ? "Sending last batch ..." : "Sending batch of 300 log records...";
CLIO.out(beginMessage);
diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
index e6ff82d21d4..8a92e6801e9 100755
--- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
@@ -237,6 +237,7 @@ public class SolrCLI implements CLIO {
else if ("export".equals(toolType)) return new ExportTool();
else if ("package".equals(toolType)) return new PackageTool();
else if ("post".equals(toolType)) return new PostTool();
+ else if ("postlogs".equals(toolType)) return new PostLogsTool();
else if ("version".equals(toolType)) return new VersionTool();
// If you add a built-in tool to this class, add it here to avoid
diff --git a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java b/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java
index 8ee87a3b8a8..cf4fee63b35 100644
--- a/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/SolrLogPostTool.java
@@ -16,30 +16,12 @@
*/
package org.apache.solr.cli;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.LineNumberReader;
-import java.net.URLDecoder;
-import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.UUID;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.impl.Http2SolrClient;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.SolrInputField;
-import org.apache.solr.handler.component.ShardRequest;
-
-/** A command line tool for indexing Solr logs in the out-of-the-box log format. */
+/**
+ * A command line tool for indexing Solr logs in the out-of-the-box log format.
+ *
+ * @deprecated Please use {@link PostLogsTool} that is exposed as 'bin/solr postlogs'.
+ */
+@Deprecated(since = "9.4")
public class SolrLogPostTool {
public static void main(String[] args) throws Exception {
@@ -60,524 +42,9 @@ public class SolrLogPostTool {
CLIO.out("");
return;
}
-
String baseUrl = args[0];
String root = args[1];
-
- Http2SolrClient.Builder builder = new Http2SolrClient.Builder(baseUrl);
- try (SolrClient client = builder.build()) {
- int rec = 0;
- UpdateRequest request = new UpdateRequest();
-
- List<Path> files;
- try (Stream<Path> stream = Files.walk(Path.of(root), Integer.MAX_VALUE)) {
- files = stream.filter(Files::isRegularFile).collect(Collectors.toList());
- }
-
- for (Path file : files) {
- try (LineNumberReader reader =
- new LineNumberReader(Files.newBufferedReader(file, Charset.defaultCharset()))) {
- LogRecordReader recordReader = new LogRecordReader(reader);
- SolrInputDocument doc;
- String fileName = file.getFileName().toString();
- while (true) {
- try {
- doc = recordReader.readRecord();
- } catch (Throwable t) {
- CLIO.err(
- "Error reading log record:" + reader.getLineNumber() + " from file:" + fileName);
- CLIO.err(t.getMessage());
- continue;
- }
-
- if (doc == null) {
- break;
- }
-
- rec++;
- UUID id = UUID.randomUUID();
- doc.setField("id", id.toString());
- doc.setField("file_s", fileName);
- request.add(doc);
- if (rec == 300) {
- sendBatch(client, request, false /* normal batch */);
- request = new UpdateRequest();
- rec = 0;
- }
- }
- }
- }
-
- if (rec > 0) {
- sendBatch(client, request, true /* last batch */);
- }
- }
- }
-
- private static void sendBatch(SolrClient client, UpdateRequest request, boolean lastRequest) {
- final String beginMessage =
- lastRequest ? "Sending last batch ..." : "Sending batch of 300 log records...";
- CLIO.out(beginMessage);
- try {
- request.process(client);
- CLIO.out("Batch sent");
- } catch (Exception e) {
- CLIO.err("Batch sending failed: " + e.getMessage());
- e.printStackTrace(CLIO.getErrStream());
- }
-
- if (lastRequest) {
- try {
- client.commit();
- CLIO.out("Committed");
- } catch (Exception e) {
- CLIO.err("Unable to commit documents: " + e.getMessage());
- e.printStackTrace(CLIO.getErrStream());
- }
- }
- }
-
- public static class LogRecordReader {
-
- private BufferedReader bufferedReader;
- private String pushedBack = null;
- private boolean finished = false;
- private String cause;
- private Pattern p =
- Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d[\\s|T]\\d\\d:\\d\\d\\:\\d\\d.\\d\\d\\d)");
- private Pattern minute =
- Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d[\\s|T]\\d\\d:\\d\\d)");
- private Pattern tenSecond =
- Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d[\\s|T]\\d\\d:\\d\\d:\\d)");
-
- public LogRecordReader(BufferedReader bufferedReader) throws IOException {
- this.bufferedReader = bufferedReader;
- }
-
- public SolrInputDocument readRecord() throws IOException {
- while (true) {
- String line = null;
-
- if (finished) {
- return null;
- }
-
- if (pushedBack != null) {
- line = pushedBack;
- pushedBack = null;
- } else {
- line = bufferedReader.readLine();
- }
-
- if (line != null) {
- SolrInputDocument lineDoc = new SolrInputDocument();
- String date = parseDate(line);
- String minute = parseMinute(line);
- String tenSecond = parseTenSecond(line);
- lineDoc.setField("date_dt", date);
- lineDoc.setField("time_minute_s", minute);
- lineDoc.setField("time_ten_second_s", tenSecond);
- lineDoc.setField("line_t", line);
- lineDoc.setField("type_s", "other"); // Overridden by known types below
-
- if (line.contains("Registered new searcher")) {
- parseNewSearch(lineDoc, line);
- } else if (line.contains("path=/update")) {
- parseUpdate(lineDoc, line);
- } else if (line.contains(" ERROR ")) {
- this.cause = null;
- parseError(lineDoc, line, readTrace());
- } else if (line.contains("QTime=")) {
- parseQueryRecord(lineDoc, line);
- }
-
- return lineDoc;
- } else {
- return null;
- }
- }
- }
-
- private String readTrace() throws IOException {
- StringBuilder buf = new StringBuilder();
- buf.append("%html ");
-
- while (true) {
- String line = bufferedReader.readLine();
- if (line == null) {
- finished = true;
- return buf.toString();
- } else {
- // look for a date at the beginning of the line
- // If it's not there then read into the stack trace buffer
- Matcher m = p.matcher(line);
-
- if (!m.find() && buf.length() < 10000) {
- // Line does not start with a timestamp so append to the stack trace
- buf.append(line.replace("\t", " ")).append("<br/>");
- if (line.startsWith("Caused by:")) {
- this.cause = line;
- }
- } else {
- pushedBack = line;
- break;
- }
- }
- }
-
- return buf.toString();
- }
-
- private String parseDate(String line) {
- Matcher m = p.matcher(line);
- if (m.find()) {
- String date = m.group(1);
- return date.replace(" ", "T") + "Z";
- }
-
- return null;
- }
-
- private String parseMinute(String line) {
- Matcher m = minute.matcher(line);
- if (m.find()) {
- String date = m.group(1);
- return date.replace(" ", "T") + ":00Z";
- }
-
- return null;
- }
-
- private String parseTenSecond(String line) {
- Matcher m = tenSecond.matcher(line);
- if (m.find()) {
- String date = m.group(1);
- return date.replace(" ", "T") + "0Z";
- }
-
- return null;
- }
-
- private void setFieldIfUnset(SolrInputDocument doc, String fieldName, String fieldValue) {
- if (doc.containsKey(fieldName)) return;
-
- doc.setField(fieldName, fieldValue);
- }
-
- private void parseError(SolrInputDocument lineRecord, String line, String trace) {
- lineRecord.setField("type_s", "error");
-
- // Don't include traces that have only the %html header.
- if (trace != null && trace.length() > 6) {
- lineRecord.setField("stack_t", trace);
- }
-
- if (this.cause != null) {
- lineRecord.setField("root_cause_t", cause.replace("Caused by:", "").trim());
- }
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
- }
-
- private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("qtime_i", parseQTime(line));
- lineRecord.setField("status_s", parseStatus(line));
-
- String path = parsePath(line);
- lineRecord.setField("path_s", path);
-
- if (line.contains("hits=")) {
- lineRecord.setField("hits_l", parseHits(line));
- }
-
- String params = parseParams(line);
- lineRecord.setField("params_t", params);
- addParams(lineRecord, params);
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("node_s", parseNode(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
-
- if (path != null && path.contains("/admin")) {
- lineRecord.setField("type_s", "admin");
- } else if (path != null && params.contains("/replication")) {
- lineRecord.setField("type_s", "replication");
- } else if (path != null && path.contains("/get")) {
- lineRecord.setField("type_s", "get");
- } else {
- lineRecord.setField("type_s", "query");
- }
- }
-
- private void parseNewSearch(SolrInputDocument lineRecord, String line) {
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("type_s", "newSearcher");
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
- }
-
- private String parseCollection(String line) {
- char[] ca = {' ', ']', ','};
- String[] parts = line.split("c:");
- if (parts.length >= 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private void parseUpdate(SolrInputDocument lineRecord, String line) {
- if (line.contains("deleteByQuery=")) {
- lineRecord.setField("type_s", "deleteByQuery");
- } else if (line.contains("delete=")) {
- lineRecord.setField("type_s", "delete");
- } else if (line.contains("commit=true")) {
- lineRecord.setField("type_s", "commit");
- } else {
- lineRecord.setField("type_s", "update");
- }
-
- lineRecord.setField("collection_s", parseCollection(line));
- lineRecord.setField("core_s", parseCore(line));
- lineRecord.setField("shard_s", parseShard(line));
- lineRecord.setField("replica_s", parseReplica(line));
- }
-
- private String parseCore(String line) {
- char[] ca = {' ', ']', '}', ','};
- String[] parts = line.split("x:");
- if (parts.length >= 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseShard(String line) {
- char[] ca = {' ', ']', '}', ','};
- String[] parts = line.split("s:");
- if (parts.length >= 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseReplica(String line) {
- char[] ca = {' ', ']', '}', ','};
- String[] parts = line.split("r:");
- if (parts.length >= 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parsePath(String line) {
- char[] ca = {' '};
- String[] parts = line.split(" path=");
- if (parts.length == 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseQTime(String line) {
- char[] ca = {'\n', '\r'};
- String[] parts = line.split(" QTime=");
- if (parts.length == 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseNode(String line) {
- char[] ca = {' ', ']', '}', ','};
- String[] parts = line.split("node_name=n:");
- if (parts.length >= 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseStatus(String line) {
- char[] ca = {' ', '\n', '\r'};
- String[] parts = line.split(" status=");
- if (parts.length == 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseHits(String line) {
- char[] ca = {' '};
- String[] parts = line.split(" hits=");
- if (parts.length == 2) {
- return readUntil(parts[1], ca);
- } else {
- return null;
- }
- }
-
- private String parseParams(String line) {
- char[] ca = {' '};
- String[] parts = line.split(" params=");
- if (parts.length == 2) {
- String p = readUntil(parts[1].substring(1), ca);
- return p.substring(0, p.length() - 1);
- } else {
- return null;
- }
- }
-
- private String readUntil(String s, char[] chars) {
- StringBuilder builder = new StringBuilder();
- for (int i = 0; i < s.length(); i++) {
- char a = s.charAt(i);
- for (char c : chars) {
- if (a == c) {
- return builder.toString();
- }
- }
- builder.append(a);
- }
-
- return builder.toString();
- }
-
- private void addParams(SolrInputDocument doc, String params) {
- String[] pairs = params.split("&");
- for (String pair : pairs) {
- String[] parts = pair.split("=");
- if (parts.length == 2 && parts[0].equals("q")) {
- String dq = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "q_s", dq);
- setFieldIfUnset(doc, "q_t", dq);
- }
-
- if (parts[0].equals("rows")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "rows_i", dr);
- }
-
- if (parts[0].equals("start")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "start_i", dr);
- }
-
- if (parts[0].equals("distrib")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "distrib_s", dr);
- }
-
- if (parts[0].equals("shards")) {
- setFieldIfUnset(doc, "shards_s", "true");
- }
-
- if (parts[0].equals("ids") && !isRTGRequest(doc)) {
- setFieldIfUnset(doc, "ids_s", "true");
- }
-
- if (parts[0].equals("isShard")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "isShard_s", dr);
- }
-
- if (parts[0].equals("wt")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "wt_s", dr);
- }
-
- if (parts[0].equals("facet")) {
- String dr = URLDecoder.decode(parts[1], Charset.defaultCharset());
- setFieldIfUnset(doc, "facet_s", dr);
- }
-
- if (parts[0].equals("shards.purpose")) {
- try {
- int purpose = Integer.parseInt(parts[1]);
- String[] purposes = getRequestPurposeNames(purpose);
- for (String p : purposes) {
- doc.addField("purpose_ss", p);
- }
- } catch (Throwable e) {
- // We'll just sit on this for now and not interrupt the load for this one field.
- }
- }
- }
-
- // Special params used to determine what stage a query is.
- // So we populate with defaults.
- // The absence of the distrib params means it's a distributed query.
- setFieldIfUnset(doc, "distrib_s", "true");
- setFieldIfUnset(doc, "shards_s", "false");
- setFieldIfUnset(doc, "ids_s", "false");
- }
-
- private boolean isRTGRequest(SolrInputDocument doc) {
- final SolrInputField path = doc.getField("path_s");
- if (path == null) return false;
-
- return "/get".equals(path.getValue());
- }
- }
-
- private static final Map<Integer, String> purposes;
- protected static final String UNKNOWN_VALUE = "Unknown";
- private static final String[] purposeUnknown = new String[] {UNKNOWN_VALUE};
-
- public static String[] getRequestPurposeNames(Integer reqPurpose) {
- if (reqPurpose != null) {
- int valid = 0;
- for (Map.Entry<Integer, String> entry : purposes.entrySet()) {
- if ((reqPurpose & entry.getKey()) != 0) {
- valid++;
- }
- }
- if (valid == 0) {
- return purposeUnknown;
- } else {
- String[] result = new String[valid];
- int i = 0;
- for (Map.Entry<Integer, String> entry : purposes.entrySet()) {
- if ((reqPurpose & entry.getKey()) != 0) {
- result[i] = entry.getValue();
- i++;
- }
- }
- return result;
- }
- }
- return purposeUnknown;
- }
-
- static {
- Map<Integer, String> map = new TreeMap<>();
- map.put(ShardRequest.PURPOSE_PRIVATE, "PRIVATE");
- map.put(ShardRequest.PURPOSE_GET_TOP_IDS, "GET_TOP_IDS");
- map.put(ShardRequest.PURPOSE_REFINE_TOP_IDS, "REFINE_TOP_IDS");
- map.put(ShardRequest.PURPOSE_GET_FACETS, "GET_FACETS");
- map.put(ShardRequest.PURPOSE_REFINE_FACETS, "REFINE_FACETS");
- map.put(ShardRequest.PURPOSE_GET_FIELDS, "GET_FIELDS");
- map.put(ShardRequest.PURPOSE_GET_HIGHLIGHTS, "GET_HIGHLIGHTS");
- map.put(ShardRequest.PURPOSE_GET_DEBUG, "GET_DEBUG");
- map.put(ShardRequest.PURPOSE_GET_STATS, "GET_STATS");
- map.put(ShardRequest.PURPOSE_GET_TERMS, "GET_TERMS");
- map.put(ShardRequest.PURPOSE_GET_TOP_GROUPS, "GET_TOP_GROUPS");
- map.put(ShardRequest.PURPOSE_GET_MLT_RESULTS, "GET_MLT_RESULTS");
- map.put(ShardRequest.PURPOSE_REFINE_PIVOT_FACETS, "REFINE_PIVOT_FACETS");
- map.put(ShardRequest.PURPOSE_SET_TERM_STATS, "SET_TERM_STATS");
- map.put(ShardRequest.PURPOSE_GET_TERM_STATS, "GET_TERM_STATS");
- purposes = Collections.unmodifiableMap(map);
+ PostLogsTool postLogsTool = new PostLogsTool();
+ postLogsTool.runCommand(baseUrl, root);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cli/SolrLogPostToolTest.java b/solr/core/src/test/org/apache/solr/cli/PostLogsToolTest.java
similarity index 98%
rename from solr/core/src/test/org/apache/solr/cli/SolrLogPostToolTest.java
rename to solr/core/src/test/org/apache/solr/cli/PostLogsToolTest.java
index 2ebfb9f1b82..6240c5443b1 100644
--- a/solr/core/src/test/org/apache/solr/cli/SolrLogPostToolTest.java
+++ b/solr/core/src/test/org/apache/solr/cli/PostLogsToolTest.java
@@ -22,20 +22,19 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.cli.SolrLogPostTool.LogRecordReader;
+import org.apache.solr.cli.PostLogsTool.LogRecordReader;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.junit.BeforeClass;
import org.junit.Test;
-public class SolrLogPostToolTest extends SolrTestCaseJ4 {
+public class PostLogsToolTest extends SolrTestCaseJ4 {
private static boolean solr9Format;
@BeforeClass
public static void beforeClass() {
solr9Format = random().nextBoolean();
- System.out.println("Solr 9 Format: " + solr9Format);
}
private String sometimesSolr9Format(String record) {
@@ -310,7 +309,7 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
assertEquals(collection.getValue(), "test");
}
- // Ensure SolrLogPostTool parses _all_ log lines into searchable records
+ // Ensure PostLogsTool parses _all_ log lines into searchable records
@Test
public void testOtherRecord() throws Exception {
final String record =
diff --git a/solr/packaging/test/test_postlogs.bats b/solr/packaging/test/test_postlogs.bats
index 32f7c379a7c..64c8230fefd 100644
--- a/solr/packaging/test/test_postlogs.bats
+++ b/solr/packaging/test/test_postlogs.bats
@@ -38,7 +38,7 @@ teardown() {
delete_all_collections
}
-@test "post solr log into solr" {
+@test "post solr log into solr via script" {
run solr create_collection -c COLL_NAME
assert_output --partial "Created collection 'COLL_NAME'"
@@ -49,3 +49,15 @@ teardown() {
run curl 'http://localhost:8983/solr/COLL_NAME/select?q=*:*'
refute_output --partial '"numFound":0'
}
+
+@test "post solr log into solr via cli" {
+ run solr create_collection -c COLL_NAME
+ assert_output --partial "Created collection 'COLL_NAME'"
+
+ run solr postlogs -url http://localhost:8983/solr/COLL_NAME -rootdir ${SOLR_LOGS_DIR}/solr.log
+ assert_output --partial 'Sending last batch'
+ assert_output --partial 'Committed'
+
+ run curl 'http://localhost:8983/solr/COLL_NAME/select?q=*:*'
+ refute_output --partial '"numFound":0'
+}
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc b/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc
index 7b9412570bd..7ac02556335 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/logs.adoc
@@ -24,25 +24,41 @@ See the xref:math-start.adoc[] chapter to learn how to get started with visualiz
== Loading
-The out-of-the-box Solr log format can be loaded into a Solr index using the `bin/postlogs` command line tool located in the `bin/` directory of the Solr distribution.
+The out-of-the-box Solr log format can be loaded into a Solr index using the `bin/solr postlogs` command line tool located in the `bin/` directory of the Solr distribution.
-NOTE: If working from the source distribution the
-distribution must first be built before `postlogs` can be run.
+=== Postlogs
-The `postlogs` script is designed to be run from the root directory of the Solr distribution.
+The `postlogs` command reads in Solr's log format and indexes it in a Solr collection.
-The `postlogs` script takes two parameters:
+`bin/solr postlogs [options]`
-* Solr base URL (with collection): `http://localhost:8983/solr/logs`
-* File path to root of the logs directory: All files found under this directory (including sub-directories) will be indexed.
+`bin/solr postlogs -help`
+
+==== Healthcheck Parameters
+
+`-url <ADDRESS>`::
++
+[%autowidth,frame=none]
+|===
+|Required |Default: none
+|===
++
+Address of the collection, example http://localhost:8983/solr/collection1/.
++
+
+`-rootdir <DIRECTORY>`::
++
+[%autowidth,frame=none]
+|===
+|Required |Default: none
+|===
++
+File path to root of the logs directory: All files found under this directory (including sub-directories) will be indexed.
If the path points to a single log file only that log file will be loaded.
-Below is a sample execution of the `postlogs` tool:
++
+*Example*: `bin/solr postlogs --url http://localhost:8983/solr/logs --rootdir /var/logs/solrlogs`
-[source,text]
-----
-./bin/postlogs http://localhost:8983/solr/logs /var/logs/solrlogs
-----
The example above will index all the log files under `/var/logs/solrlogs` to the `logs` collection found at the base url `http://localhost:8983/solr`.