You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ge...@apache.org on 2020/06/16 12:21:43 UTC
[lucene-solr] branch master updated: SOLR-14558: Record all log
lines in SolrLogPostTool (#1570)
This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new a7792b1 SOLR-14558: Record all log lines in SolrLogPostTool (#1570)
a7792b1 is described below
commit a7792b129b096245d70ab960f57d15842c60bbd6
Author: Jason Gerlowski <ge...@apache.org>
AuthorDate: Tue Jun 16 08:21:24 2020 -0400
SOLR-14558: Record all log lines in SolrLogPostTool (#1570)
---
.../java/org/apache/solr/util/SolrLogPostTool.java | 132 +++++++++------------
.../org/apache/solr/util/SolrLogPostToolTest.java | 16 +++
2 files changed, 71 insertions(+), 77 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java b/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
index 83d0449..7f3a567 100644
--- a/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
@@ -186,20 +186,25 @@ public class SolrLogPostTool {
}
if (line != null) {
+ SolrInputDocument lineDoc = new SolrInputDocument();
+ lineDoc.setField("date_dt", parseDate(line));
+ lineDoc.setField("line_t", line);
+ lineDoc.setField("type_s", "other"); // Overridden by known types below
+
if (line.contains("Registered new searcher")) {
- return parseNewSearch(line);
+ parseNewSearch(lineDoc, line);
} else if (line.contains("path=/update")) {
- return parseUpdate(line);
+ parseUpdate(lineDoc, line);
} else if (line.contains(" ERROR ")) {
this.cause = null;
- return parseError(line, readTrace());
+ parseError(lineDoc, line, readTrace());
} else if (line.contains("start commit")) {
- return parseCommit(line);
+ parseCommit(lineDoc, line);
} else if(line.contains("QTime=")) {
- return parseQueryRecord(line);
- } else {
- continue;
+ parseQueryRecord(lineDoc, line);
}
+
+ return lineDoc;
} else {
return null;
}
@@ -252,94 +257,73 @@ public class SolrLogPostTool {
doc.setField(fieldName, fieldValue);
}
- private SolrInputDocument parseError(String line, String trace) throws IOException {
- SolrInputDocument doc = new SolrInputDocument();
- doc.setField("date_dt", parseDate(line));
- doc.setField("type_s", "error");
- doc.setField("line_t", line);
+ private void parseError(SolrInputDocument lineRecord, String line, String trace) throws IOException {
+ lineRecord.setField("type_s", "error");
//Don't include traces that have only the %html header.
if(trace != null && trace.length() > 6) {
- doc.setField("stack_t", trace);
+ lineRecord.setField("stack_t", trace);
}
if(this.cause != null) {
- doc.setField("root_cause_t", cause.replace("Caused by:", "").trim());
+ lineRecord.setField("root_cause_t", cause.replace("Caused by:", "").trim());
}
- doc.setField("collection_s", parseCollection(line));
- doc.setField("core_s", parseCore(line));
- doc.setField("shard_s", parseShard(line));
- doc.setField("replica_s", parseReplica(line));
-
- return doc;
+ lineRecord.setField("collection_s", parseCollection(line));
+ lineRecord.setField("core_s", parseCore(line));
+ lineRecord.setField("shard_s", parseShard(line));
+ lineRecord.setField("replica_s", parseReplica(line));
}
- private SolrInputDocument parseCommit(String line) throws IOException {
- SolrInputDocument doc = new SolrInputDocument();
- doc.setField("date_dt", parseDate(line));
- doc.setField("type_s", "commit");
- doc.setField("line_t", line);
- doc.setField("soft_commit_s", Boolean.toString(line.contains("softCommit=true")));
-
- doc.setField("open_searcher_s", Boolean.toString(line.contains("openSearcher=true")));
+ private void parseCommit(SolrInputDocument lineRecord, String line) throws IOException {
+ lineRecord.setField("type_s", "commit");
+ lineRecord.setField("soft_commit_s", Boolean.toString(line.contains("softCommit=true")));
- doc.setField("collection_s", parseCollection(line));
- doc.setField("core_s", parseCore(line));
- doc.setField("shard_s", parseShard(line));
- doc.setField("replica_s", parseReplica(line));
+ lineRecord.setField("open_searcher_s", Boolean.toString(line.contains("openSearcher=true")));
- return doc;
+ lineRecord.setField("collection_s", parseCollection(line));
+ lineRecord.setField("core_s", parseCore(line));
+ lineRecord.setField("shard_s", parseShard(line));
+ lineRecord.setField("replica_s", parseReplica(line));
}
- private SolrInputDocument parseQueryRecord(String line) {
-
- SolrInputDocument doc = new SolrInputDocument();
- doc.setField("date_dt", parseDate(line));
- doc.setField("qtime_i", parseQTime(line));
- doc.setField("status_s", parseStatus(line));
+ private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
+ lineRecord.setField("qtime_i", parseQTime(line));
+ lineRecord.setField("status_s", parseStatus(line));
String path = parsePath(line);
- doc.setField("path_s", path);
+ lineRecord.setField("path_s", path);
if(line.contains("hits=")) {
- doc.setField("hits_l", parseHits(line));
+ lineRecord.setField("hits_l", parseHits(line));
}
String params = parseParams(line);
- doc.setField("params_t", params);
- addParams(doc, params);
+ lineRecord.setField("params_t", params);
+ addParams(lineRecord, params);
- doc.setField("collection_s", parseCollection(line));
- doc.setField("core_s", parseCore(line));
- doc.setField("node_s", parseNode(line));
- doc.setField("shard_s", parseShard(line));
- doc.setField("replica_s", parseReplica(line));
+ lineRecord.setField("collection_s", parseCollection(line));
+ lineRecord.setField("core_s", parseCore(line));
+ lineRecord.setField("node_s", parseNode(line));
+ lineRecord.setField("shard_s", parseShard(line));
+ lineRecord.setField("replica_s", parseReplica(line));
if(path != null && path.contains("/admin")) {
- doc.setField("type_s", "admin");
+ lineRecord.setField("type_s", "admin");
} else if(path != null && params.contains("/replication")) {
- doc.setField("type_s", "replication");
+ lineRecord.setField("type_s", "replication");
} else if (path != null && path.contains("/get")) {
- doc.setField("type_s", "get");
+ lineRecord.setField("type_s", "get");
} else {
- doc.setField("type_s", "query");
+ lineRecord.setField("type_s", "query");
}
-
- return doc;
}
- private SolrInputDocument parseNewSearch(String line) {
-
- SolrInputDocument doc = new SolrInputDocument();
- doc.setField("date_dt", parseDate(line));
- doc.setField("core_s", parseNewSearcherCore(line));
- doc.setField("type_s", "newSearcher");
- doc.setField("line_t", line);
-
- return doc;
+ private void parseNewSearch(SolrInputDocument lineRecord, String line) {
+ lineRecord.setField("core_s", parseNewSearcherCore(line));
+ lineRecord.setField("type_s", "newSearcher");
}
private String parseCollection(String line) {
@@ -352,25 +336,19 @@ public class SolrLogPostTool {
}
}
- private SolrInputDocument parseUpdate(String line) {
- SolrInputDocument doc = new SolrInputDocument();
- doc.setField("date_dt", parseDate(line));
-
+ private void parseUpdate(SolrInputDocument lineRecord, String line) {
if(line.contains("deleteByQuery=")) {
- doc.setField("type_s", "deleteByQuery");
+ lineRecord.setField("type_s", "deleteByQuery");
} else if(line.contains("delete=")) {
- doc.setField("type_s", "delete");
+ lineRecord.setField("type_s", "delete");
} else {
- doc.setField("type_s", "update");
+ lineRecord.setField("type_s", "update");
}
- doc.setField("collection_s", parseCollection(line));
- doc.setField("core_s", parseCore(line));
- doc.setField("shard_s", parseShard(line));
- doc.setField("replica_s", parseReplica(line));
- doc.setField("line_t", line);
-
- return doc;
+ lineRecord.setField("collection_s", parseCollection(line));
+ lineRecord.setField("core_s", parseCore(line));
+ lineRecord.setField("shard_s", parseShard(line));
+ lineRecord.setField("replica_s", parseReplica(line));
}
private String parseNewSearcherCore(String line) {
diff --git a/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java b/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
index c65f5f1..390c8ac 100644
--- a/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
+++ b/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
@@ -18,6 +18,7 @@ package org.apache.solr.util;
import java.io.BufferedReader;
import java.io.StringReader;
+import java.util.Collection;
import java.util.List;
import java.util.ArrayList;
@@ -286,6 +287,21 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
assertEquals(core.getValue(), "production_cv_month_201912_shard35_replica_n1");
}
+ // Ensure SolrLogPostTool parses _all_ log lines into searchable records
+ @Test
+ public void testOtherRecord() throws Exception {
+ final String record = "2020-06-11 11:59:08.386 INFO (main) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)";
+ final List<SolrInputDocument> docs = readDocs(record);
+ assertEquals(docs.size(), 1);
+
+ SolrInputDocument doc = docs.get(0);
+ final Collection<String> fields = doc.getFieldNames();
+ assertEquals(3, fields.size());
+ assertEquals("2020-06-11T11:59:08.386", doc.getField("date_dt").getValue());
+ assertEquals("other", doc.getField("type_s").getValue());
+ assertEquals(record, doc.getField("line_t").getValue());
+ }
+
private List<SolrInputDocument> readDocs(String records) throws Exception {
BufferedReader bufferedReader = new BufferedReader(new StringReader(records));
ArrayList<SolrInputDocument> list = new ArrayList();