You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2019/12/09 19:49:55 UTC
[lucene-solr] branch SOLR-13635-solrlogstream updated (d7f7b6a ->
4e49feb)
This is an automated email from the ASF dual-hosted git repository.
jbernste pushed a change to branch SOLR-13635-solrlogstream
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from d7f7b6a SOLR-13635: Add SolrLogStream Streaming Expressions
new 3594c78 SOLR-13635: Add basic support for parsing searcher openings and updates
new 4e49feb SOLR-13635: Add stacktraces
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../solr/client/solrj/io/stream/SolrLogStream.java | 127 ++++++++++++++++++++-
1 file changed, 121 insertions(+), 6 deletions(-)
[lucene-solr] 02/02: SOLR-13635: Add stacktraces
Posted by jb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jbernste pushed a commit to branch SOLR-13635-solrlogstream
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 4e49febbab534d7db2ea162f6ca58691eac76589
Author: Joel Bernstein <jb...@apache.org>
AuthorDate: Mon Dec 9 14:49:34 2019 -0500
SOLR-13635: Add stacktraces
---
.../solr/client/solrj/io/stream/SolrLogStream.java | 95 ++++++++++++++++++++--
1 file changed, 87 insertions(+), 8 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrLogStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrLogStream.java
index db5a187..2c3be9d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrLogStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrLogStream.java
@@ -37,9 +37,8 @@ public class SolrLogStream extends TupleStream implements Expressible {
private static final long serialVersionUID = 1;
private String currentFile;
private int lineNumber;
- private Pattern handler;
- protected TupleStream originalStream;
+ protected PushBackStream originalStream;
public SolrLogStream(StreamExpression expression,StreamFactory factory) throws IOException {
// grab all parameters out
@@ -58,8 +57,7 @@ public class SolrLogStream extends TupleStream implements Expressible {
}
private void init(TupleStream stream) throws IOException{
- handler = Pattern.compile("collection=c:([A-Za-z]),");
- this.originalStream = stream;
+ this.originalStream = new PushBackStream(stream);
}
@Override
@@ -141,6 +139,10 @@ public class SolrLogStream extends TupleStream implements Expressible {
t = parseNewSearch(line);
} else if(line.contains("path=/update")) {
t = parseUpdate(line);
+ } else if(line.contains(" ERROR ")) {
+ t = parseError(line);
+ } else if(line.contains("start commit")) {
+ t=parseCommit(line);
} else {
continue;
}
@@ -164,6 +166,67 @@ public class SolrLogStream extends TupleStream implements Expressible {
}
}
+
+ private Tuple parseError(String line) throws IOException {
+ String[] parts = line.split("\\s+");
+ Tuple tuple = new Tuple();
+ tuple.put("date_dt", parts[0]);
+ tuple.put("type_s", "error");
+ tuple.put("line_t", line);
+ StringBuilder buf = new StringBuilder("%html ");
+ //Check to see if there is a stack trace;
+ while(true) {
+ Tuple t = originalStream.read();
+ if(t.EOF) {
+ originalStream.pushBack(t);
+ break;
+ } else{
+ if(t.getString("line").startsWith("\t")) {
+ buf.append(t.getString("line").replace("\t", "")+"<br/>");
+ } else {
+ originalStream.pushBack(t);
+ break;
+ }
+ }
+ }
+
+ if(buf.length() > 7) {
+ tuple.put("stack_t", buf.toString());
+ }
+
+ tuple.put("collection_s", parseCollection(line));
+ tuple.put("core_s", parseCore(line));
+ tuple.put("node_s", parseNode(line));
+
+ return tuple;
+ }
+
+
+ private Tuple parseCommit(String line) throws IOException {
+ String[] parts = line.split("\\s+");
+ Tuple tuple = new Tuple();
+ tuple.put("date_dt", parts[0]);
+ tuple.put("type_s", "commit");
+ tuple.put("line_t", line);
+ if(line.contains("softCommit=true")) {
+ tuple.put("soft_commit_s", "true");
+ } else {
+ tuple.put("soft_commit_s", "false");
+ }
+
+ if(line.contains("openSearcher=true")) {
+ tuple.put("open_searcher_s", "true");
+ } else {
+ tuple.put("open_searcher_s", "false");
+ }
+
+ tuple.put("collection_s", parseCollection(line));
+ tuple.put("core_s", parseCore(line));
+ tuple.put("node_s", parseNode(line));
+
+ return tuple;
+ }
+
private Tuple parseQueryRecord(String line) {
String[] parts = line.split("\\s+");
@@ -171,7 +234,7 @@ public class SolrLogStream extends TupleStream implements Expressible {
tuple.put("date_dt", parts[0]);
String qtime = parts[parts.length-1];
tuple.put("qtime_i", qtime.split("=")[1]);
-
+ tuple.put("type_s", "query");
String status = parts[parts.length-2];
tuple.put("status_s", status.split("=")[1]);
@@ -182,7 +245,6 @@ public class SolrLogStream extends TupleStream implements Expressible {
String params = parts[parts.length-4];
tuple.put("params_t", params.substring(7));
addParams(tuple, params);
-
}
@@ -203,6 +265,13 @@ public class SolrLogStream extends TupleStream implements Expressible {
tuple.put("collection_s", parseCollection(line));
tuple.put("core_s", parseCore(line));
tuple.put("node_s", parseNode(line));
+ String path = parsePath(line);
+ tuple.put("path_s", path);
+ if(path != null && path.contains("/admin")) {
+ tuple.put("type_s", "admin");
+ } else {
+ tuple.put("type_s", "query");
+ }
return tuple;
}
@@ -213,7 +282,7 @@ public class SolrLogStream extends TupleStream implements Expressible {
Tuple tuple = new Tuple();
tuple.put("date_dt", parts[0]);
tuple.put("core_s", parseNewSearcherCollection(line));
- tuple.put("new_searcher", "true");
+ tuple.put("type_s", "newSearcher");
return tuple;
}
@@ -232,7 +301,7 @@ public class SolrLogStream extends TupleStream implements Expressible {
String[] parts = line.split("\\s+");
Tuple tuple = new Tuple();
tuple.put("date_dt", parts[0]);
- tuple.put("update_s", "true");
+ tuple.put("type_s", "update");
tuple.put("collection_s", parseCollection(line));
tuple.put("core_s", parseCore(line));
tuple.put("node_s", parseNode(line));
@@ -259,6 +328,16 @@ public class SolrLogStream extends TupleStream implements Expressible {
}
}
+ private String parsePath(String line) {
+ char[] ca = {' '};
+ String parts[] = line.split(" path=");
+ if(parts.length == 2) {
+ return readUntil(parts[1], ca);
+ } else {
+ return null;
+ }
+ }
+
private String parseNode(String line) {
char[] ca = {',', '}'};
String parts[] = line.split("node_name=n:");
[lucene-solr] 01/02: SOLR-13635: Add basic support for parsing
searcher openings and updates
Posted by jb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jbernste pushed a commit to branch SOLR-13635-solrlogstream
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 3594c785cfc47ca1cb22a0f8fec4d3466f8494a2
Author: Joel Bernstein <jb...@apache.org>
AuthorDate: Wed Dec 4 13:31:31 2019 -0500
SOLR-13635: Add basic support for parsing searcher openings and updates
---
.../solr/client/solrj/io/stream/SolrLogStream.java | 36 ++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrLogStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrLogStream.java
index db9ccbc..db5a187 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrLogStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrLogStream.java
@@ -137,6 +137,10 @@ public class SolrLogStream extends TupleStream implements Expressible {
if (line.contains("QTime=")) {
t = parseQueryRecord(line);
+ } else if(line.contains("Registered new searcher")) {
+ t = parseNewSearch(line);
+ } else if(line.contains("path=/update")) {
+ t = parseUpdate(line);
} else {
continue;
}
@@ -203,6 +207,17 @@ public class SolrLogStream extends TupleStream implements Expressible {
return tuple;
}
+ private Tuple parseNewSearch(String line) {
+
+ String[] parts = line.split("\\s+");
+ Tuple tuple = new Tuple();
+ tuple.put("date_dt", parts[0]);
+ tuple.put("core_s", parseNewSearcherCollection(line));
+ tuple.put("new_searcher", "true");
+
+ return tuple;
+ }
+
private String parseCollection(String line) {
char[] ca = {',', '}'};
String parts[] = line.split("collection=c:");
@@ -213,6 +228,27 @@ public class SolrLogStream extends TupleStream implements Expressible {
}
}
+ private Tuple parseUpdate(String line) {
+ String[] parts = line.split("\\s+");
+ Tuple tuple = new Tuple();
+ tuple.put("date_dt", parts[0]);
+ tuple.put("update_s", "true");
+ tuple.put("collection_s", parseCollection(line));
+ tuple.put("core_s", parseCore(line));
+ tuple.put("node_s", parseNode(line));
+ return tuple;
+ }
+
+ private String parseNewSearcherCollection(String line) {
+ char[] ca = {']'};
+ String parts[] = line.split("\\[");
+ if(parts.length > 3) {
+ return readUntil(parts[2], ca);
+ } else {
+ return null;
+ }
+ }
+
private String parseCore(String line) {
char[] ca = {',', '}'};
String parts[] = line.split("core=x:");