You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2020/01/03 13:50:04 UTC
[lucene-solr] 03/03: SOLR-14130: Harden parsing logic
This is an automated email from the ASF dual-hosted git repository.
jbernste pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 8df304d12ef99b8b81de985ed3a08f26c3bc3e14
Author: Joel Bernstein <jb...@apache.org>
AuthorDate: Fri Jan 3 08:46:55 2020 -0500
SOLR-14130: Harden parsing logic
---
solr/bin/postlogs | 0
.../java/org/apache/solr/util/SolrLogPostTool.java | 41 ++++++++++++----------
.../org/apache/solr/util/SolrLogPostToolTest.java | 8 -----
3 files changed, 22 insertions(+), 27 deletions(-)
diff --git a/solr/bin/postlogs b/solr/bin/postlogs
old mode 100644
new mode 100755
diff --git a/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java b/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
index aee5ffa..3f367ce 100644
--- a/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
@@ -47,7 +47,8 @@ public class SolrLogPostTool {
CLIO.out("-- baseUrl: Example http://localhost:8983/solr/collection1");
CLIO.out("-- rootDir: All files found at or below the root will be indexed.");
CLIO.out("");
- CLIO.out("Sample syntax: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs");
+ CLIO.out("Sample syntax 1: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs/solr.log");
+ CLIO.out("Sample syntax 2: ./bin/postlogs http://localhost:8983/solr/collection1 /user/foo/logs");
CLIO.out("");
return;
}
@@ -139,13 +140,12 @@ public class SolrLogPostTool {
private String pushedBack = null;
private boolean finished = false;
private String cause;
- Pattern p = Pattern.compile("^\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d");
+ private Pattern p = Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d \\d\\d:\\d\\d\\:\\d\\d.\\d\\d\\d)");
public LogRecordReader(BufferedReader bufferedReader) throws IOException {
this.bufferedReader = bufferedReader;
}
-
public SolrInputDocument readRecord() throws IOException {
while(true) {
String line = null;
@@ -196,7 +196,7 @@ public class SolrLogPostTool {
//If it's not there then read into the stack trace buffer
Matcher m = p.matcher(line);
- if (!m.find()) {
+ if (!m.find() && buf.length() < 10000) {
//Line does not start with a timestamp so append to the stack trace
buf.append(line.replace("\t", " ") + "<br/>");
if(line.startsWith("Caused by:")) {
@@ -212,14 +212,24 @@ public class SolrLogPostTool {
return buf.toString();
}
+ private String parseDate(String line) {
+ Matcher m = p.matcher(line);
+ if(m.find()) {
+ String date = m.group(1);
+ return date.replace(" ", "T");
+ }
+
+ return null;
+ }
+
private SolrInputDocument parseError(String line, String trace) throws IOException {
- String[] parts = line.split("\\s+");
SolrInputDocument doc = new SolrInputDocument();
- doc.addField("date_dt", parts[0]+"T"+parts[1]);
+ doc.addField("date_dt", parseDate(line));
doc.addField("type_s", "error");
doc.addField("line_t", line);
- if(trace != null) {
+ //Don't include traces that have only the %html header.
+ if(trace != null && trace.length() > 6) {
doc.addField("stack_t", trace);
}
@@ -237,8 +247,7 @@ public class SolrLogPostTool {
private SolrInputDocument parseCommit(String line) throws IOException {
SolrInputDocument doc = new SolrInputDocument();
- String[] parts = line.split("\\s+");
- doc.addField("date_dt", parts[0]+"T"+parts[1]);
+ doc.addField("date_dt", parseDate(line));
doc.addField("type_s", "commit");
doc.addField("line_t", line);
if(line.contains("softCommit=true")) {
@@ -263,9 +272,8 @@ public class SolrLogPostTool {
private SolrInputDocument parseQueryRecord(String line) {
- String[] parts = line.split("\\s+");
SolrInputDocument doc = new SolrInputDocument();
- doc.addField("date_dt", parts[0]+"T"+parts[1]);
+ doc.addField("date_dt", parseDate(line));
doc.addField("qtime_i", parseQTime(line));
doc.addField("status_s", parseStatus(line));
@@ -277,9 +285,6 @@ public class SolrLogPostTool {
doc.addField("params_t", params);
addParams(doc, params);
- String ll = parts[2];
- doc.addField("log_level_s", ll);
-
doc.addField("collection_s", parseCollection(line));
doc.addField("core_s", parseCore(line));
doc.addField("node_s", parseNode(line));
@@ -299,9 +304,8 @@ public class SolrLogPostTool {
private SolrInputDocument parseNewSearch(String line) {
- String[] parts = line.split("\\s+");
SolrInputDocument doc = new SolrInputDocument();
- doc.addField("date_dt", parts[0]+"T"+parts[1]);
+ doc.addField("date_dt", parseDate(line));
doc.addField("core_s", parseNewSearcherCore(line));
doc.addField("type_s", "newSearcher");
doc.addField("line_t", line);
@@ -312,7 +316,7 @@ public class SolrLogPostTool {
private String parseCollection(String line) {
char[] ca = {' ', ']'};
String parts[] = line.split("c:");
- if(parts.length == 2) {
+ if(parts.length >= 2) {
return readUntil(parts[1], ca);
} else {
return null;
@@ -320,9 +324,8 @@ public class SolrLogPostTool {
}
private SolrInputDocument parseUpdate(String line) {
- String[] parts = line.split("\\s+");
SolrInputDocument doc = new SolrInputDocument();
- doc.addField("date_dt", parts[0]+"T"+parts[1]);
+ doc.addField("date_dt", parseDate(line));
if(line.contains("deleteByQuery=")) {
doc.addField("type_s", "deleteByQuery");
diff --git a/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java b/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
index 375f719..5e296aa 100644
--- a/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
+++ b/solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
@@ -26,16 +26,10 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.util.SolrLogPostTool.LogRecordReader;
-import org.junit.Before;
import org.junit.Test;
-
public class SolrLogPostToolTest extends SolrTestCaseJ4 {
- @Before
- public void initVariousPostTools() throws Exception {
-
- }
@Test
public void testQueryRecord() throws Exception{
@@ -51,7 +45,6 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
SolrInputField hits = doc.getField("hits_l");
SolrInputField type = doc.getField("type_s");
SolrInputField status = doc.getField("status_s");
- SolrInputField loglevel = doc.getField("log_level_s");
SolrInputField shard = doc.getField("shard_s");
SolrInputField replica = doc.getField("replica_s");
SolrInputField core = doc.getField("core_s");
@@ -66,7 +59,6 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
assertEquals(hits.getValue(), "234868");
assertEquals(type.getValue(), "query");
assertEquals(status.getValue(), "0");
- assertEquals(loglevel.getValue(), "INFO");
assertEquals(shard.getValue(), "shard1");
assertEquals(replica.getValue(), "core_node2");
assertEquals(core.getValue(), "logs4_shard1_replica_n1");