You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by yo...@apache.org on 2007/10/14 20:38:55 UTC
svn commit: r584574 - in /lucene/solr/trunk: ./
src/java/org/apache/solr/common/util/ src/java/org/apache/solr/request/
Author: yonik
Date: Sun Oct 14 11:38:54 2007
New Revision: 584574
URL: http://svn.apache.org/viewvc?rev=584574&view=rev
Log:
speed up response writers: SOLR-377
Added:
lucene/solr/trunk/src/java/org/apache/solr/common/util/FastWriter.java (with props)
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/src/java/org/apache/solr/common/util/XML.java
lucene/solr/trunk/src/java/org/apache/solr/request/JSONResponseWriter.java
lucene/solr/trunk/src/java/org/apache/solr/request/PythonResponseWriter.java
lucene/solr/trunk/src/java/org/apache/solr/request/RubyResponseWriter.java
lucene/solr/trunk/src/java/org/apache/solr/request/TextResponseWriter.java
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=584574&r1=584573&r2=584574&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Sun Oct 14 11:38:54 2007
@@ -150,6 +150,8 @@
4. SOLR-354: Optimize removing all documents. Now when a delete by query
of *:* is issued, the current index is removed. (yonik)
+ 5. SOLR-377: Speed up response writers. (yonik)
+
Bug Fixes
1. Make TextField respect sortMissingFirst and sortMissingLast fields.
(J.J. Larrea via yonik)
Added: lucene/solr/trunk/src/java/org/apache/solr/common/util/FastWriter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/common/util/FastWriter.java?rev=584574&view=auto
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/common/util/FastWriter.java (added)
+++ lucene/solr/trunk/src/java/org/apache/solr/common/util/FastWriter.java Sun Oct 14 11:38:54 2007
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.io.Writer;
+import java.io.IOException;
+
+/** Single threaded BufferedWriter
+ * Internal Solr use only, subject to change.
+ */
+public class FastWriter extends Writer {
+ // use default BUFSIZE of BufferedWriter so if we wrap that
+ // it won't cause double buffering.
+ private static final int BUFSIZE = 8192;
+ private final Writer sink;
+ private final char[] buf;
+ private int pos;
+
+ public FastWriter(Writer w) {
+ this(w, new char[BUFSIZE], 0);
+ }
+
+ public FastWriter(Writer sink, char[] tempBuffer, int start) {
+ this.sink = sink;
+ this.buf = tempBuffer;
+ this.pos = start;
+ }
+
+ public static FastWriter wrap(Writer sink) {
+ return (sink instanceof FastWriter) ? (FastWriter)sink : new FastWriter(sink);
+ }
+
+ @Override
+ public void write(int c) throws IOException {
+ write((char)c);
+ }
+
+ public void write(char c) throws IOException {
+ if (pos >= buf.length) {
+ sink.write(buf,0,pos);
+ pos=0;
+ }
+ buf[pos++] = (char)c;
+ }
+
+ @Override
+ public FastWriter append(char c) throws IOException {
+ if (pos >= buf.length) {
+ sink.write(buf,0,pos);
+ pos=0;
+ }
+ buf[pos++] = (char)c;
+ return this;
+ }
+
+ @Override
+ public void write(char cbuf[], int off, int len) throws IOException {
+ int space = buf.length - pos;
+ if (len < space) {
+ System.arraycopy(cbuf, off, buf, pos, len);
+ pos += len;
+ } else if (len<BUFSIZE) {
+ // if the data to write is small enough, buffer it.
+ System.arraycopy(cbuf, off, buf, pos, space);
+ sink.write(buf, 0, buf.length);
+ pos = len-space;
+ System.arraycopy(cbuf, off+space, buf, 0, pos);
+ } else {
+ sink.write(buf,0,pos); // flush
+ pos=0;
+ // don't buffer, just write to sink
+ sink.write(cbuf, off, len);
+ }
+ }
+
+ @Override
+ public void write(String str, int off, int len) throws IOException {
+ int space = buf.length - pos;
+ if (len < space) {
+ str.getChars(off, off+len, buf, pos);
+ pos += len;
+ } else if (len<BUFSIZE) {
+ // if the data to write is small enough, buffer it.
+ str.getChars(off, off+space, buf, pos);
+ sink.write(buf, 0, buf.length);
+ str.getChars(off+space, off+len, buf, 0);
+ pos = len-space;
+ } else {
+ sink.write(buf,0,pos); // flush
+ pos=0;
+ // don't buffer, just write to sink
+ sink.write(str, off, len);
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ sink.write(buf,0,pos);
+ pos=0;
+ sink.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ flush();
+ sink.close();
+ }
+
+ public void flushBuffer() throws IOException {
+ sink.write(buf, 0, pos);
+ pos=0;
+ }
+}
Propchange: lucene/solr/trunk/src/java/org/apache/solr/common/util/FastWriter.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/src/java/org/apache/solr/common/util/FastWriter.java
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/solr/trunk/src/java/org/apache/solr/common/util/FastWriter.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Modified: lucene/solr/trunk/src/java/org/apache/solr/common/util/XML.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/common/util/XML.java?rev=584574&r1=584573&r2=584574&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/common/util/XML.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/common/util/XML.java Sun Oct 14 11:38:54 2007
@@ -87,11 +87,13 @@
out.write('<');
out.write(tag);
if (val == null) {
- out.write("/>");
+ out.write('/');
+ out.write('>');
} else {
out.write('>');
escapeCharData(val,out);
- out.write("</");
+ out.write('<');
+ out.write('/');
out.write(tag);
out.write('>');
}
@@ -104,16 +106,19 @@
for (int i=0; i<attrs.length; i++) {
out.write(' ');
out.write(attrs[i++].toString());
- out.write("=\"");
+ out.write('=');
+ out.write('"');
out.write(attrs[i].toString());
- out.write("\"");
+ out.write('"');
}
if (val == null) {
- out.write("/>");
+ out.write('/');
+ out.write('>');
} else {
out.write('>');
out.write(val);
- out.write("</");
+ out.write('<');
+ out.write('/');
out.write(tag);
out.write('>');
}
@@ -126,16 +131,19 @@
for (int i=0; i<attrs.length; i++) {
out.write(' ');
out.write(attrs[i++].toString());
- out.write("=\"");
+ out.write('=');
+ out.write('"');
escapeAttributeValue(attrs[i].toString(), out);
- out.write("\"");
+ out.write('"');
}
if (val == null) {
- out.write("/>");
+ out.write('/');
+ out.write('>');
} else {
out.write('>');
escapeCharData(val,out);
- out.write("</");
+ out.write('<');
+ out.write('/');
out.write(tag);
out.write('>');
}
@@ -143,41 +151,16 @@
private static void escape(String str, Writer out, String[] escapes) throws IOException {
- int start=0;
- // "n" was used for counting the chars added to out...
- // removed cause it wasn't really useful so far.
- // int n=0;
-
- for (int i=start; i<str.length(); i++) {
+ for (int i=0; i<str.length(); i++) {
char ch = str.charAt(i);
- // since I already received the char, what if I put it into
- // a char array and wrote that to the stream instead of the
- // string? (would cause extra GC though)
- String subst=null;
if (ch<escapes.length) {
- subst=escapes[ch];
- }
- if (subst != null) {
- if (start<i) {
- out.write(str.substring(start,i));
- // write(str,off,len) causes problems for Jetty with chars > 127
- //out.write(str, start, i-start);
- // n+=i-start;
+ String replacement = escapes[ch];
+ if (replacement != null) {
+ out.write(replacement);
+ continue;
}
- out.write(subst);
- // n+=subst.length();
- start=i+1;
}
+ out.write(ch);
}
- if (start==0) {
- out.write(str);
- // n += str.length();
- } else if (start<str.length()) {
- out.write(str.substring(start));
- // write(str,off,len) causes problems for Jetty with chars > 127
- // out.write(str, start, str.length()-start);
- // n += str.length()-start;
- }
- // return n;
}
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/request/JSONResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/request/JSONResponseWriter.java?rev=584574&r1=584573&r2=584574&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/request/JSONResponseWriter.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/request/JSONResponseWriter.java Sun Oct 14 11:38:54 2007
@@ -81,8 +81,9 @@
}
writeNamedList(null, rsp.getValues());
if(wrapperFunction!=null) {
- writer.write(")");
+ writer.write(')');
}
+ writer.flushBuffer();
}
protected void writeKey(String fname, boolean needsEscaping) throws IOException {
@@ -496,36 +497,37 @@
escaped: quotation mark, reverse solidus, and the control
characters (U+0000 through U+001F).
*/
-
- StringBuilder sb = new StringBuilder(val.length()+8);
- sb.append('"');
+ writer.write('"');
for (int i=0; i<val.length(); i++) {
char ch = val.charAt(i);
+ if ((ch > '#' && ch != '\\') || ch==' ') { // fast path
+ writer.write(ch);
+ continue;
+ }
switch(ch) {
case '"':
case '\\':
- sb.append('\\');
- sb.append(ch);
+ writer.write('\\');
+ writer.write(ch);
break;
- case '\r': sb.append('\\').append('r'); break;
- case '\n': sb.append('\\').append('n'); break;
- case '\t': sb.append('\\').append('t'); break;
- case '\b': sb.append('\\').append('b'); break;
- case '\f': sb.append('\\').append('f'); break;
+ case '\r': writer.write('\\'); writer.write('r'); break;
+ case '\n': writer.write('\\'); writer.write('n'); break;
+ case '\t': writer.write('\\'); writer.write('t'); break;
+ case '\b': writer.write('\\'); writer.write('b'); break;
+ case '\f': writer.write('\\'); writer.write('f'); break;
// case '/':
default: {
if (ch <= 0x1F) {
- unicodeEscape(sb,ch);
+ unicodeEscape(writer,ch);
} else {
- sb.append(ch);
+ writer.write(ch);
}
}
}
}
- sb.append('"');
- writer.append(sb);
+ writer.write('"');
} else {
writer.write('"');
writer.write(val);
@@ -673,15 +675,14 @@
writeStr(name, val, false);
}
- protected static void unicodeEscape(Appendable sb, int ch) throws IOException {
- String str = Integer.toHexString(ch & 0xffff);
- switch (str.length()) {
- case 1: sb.append("\\u000"); break;
- case 2: sb.append("\\u00"); break;
- case 3: sb.append("\\u0"); break;
- default: sb.append("\\u"); break;
- }
- sb.append(str);
+ private static char[] hexdigits = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
+ protected static void unicodeEscape(Appendable out, int ch) throws IOException {
+ out.append('\\');
+ out.append('u');
+ out.append(hexdigits[(ch>>>12) ]);
+ out.append(hexdigits[(ch>>>8) & 0xf]);
+ out.append(hexdigits[(ch>>>4) & 0xf]);
+ out.append(hexdigits[(ch) & 0xf]);
}
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/request/PythonResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/request/PythonResponseWriter.java?rev=584574&r1=584573&r2=584574&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/request/PythonResponseWriter.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/request/PythonResponseWriter.java Sun Oct 14 11:38:54 2007
@@ -95,7 +95,10 @@
}
}
- writer.write( needUnicode ? "u'" : "'");
+ if (needUnicode) {
+ writer.write('u');
+ }
+ writer.write('\'');
writer.append(sb);
writer.write('\'');
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/request/RubyResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/request/RubyResponseWriter.java?rev=584574&r1=584573&r2=584574&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/request/RubyResponseWriter.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/request/RubyResponseWriter.java Sun Oct 14 11:38:54 2007
@@ -51,7 +51,8 @@
@Override
protected void writeKey(String fname, boolean needsEscaping) throws IOException {
writeStr(null, fname, needsEscaping);
- writer.write("=>");
+ writer.write('=');
+ writer.write('>');
}
@Override
@@ -63,16 +64,13 @@
// Also, there are very few escapes recognized in a single quoted string, so
// only escape the backslash and single quote.
writer.write('\'');
- // it might be more efficient to use a stringbuilder or write substrings
- // if writing chars to the stream is slow.
if (needsEscaping) {
for (int i=0; i<val.length(); i++) {
char ch = val.charAt(i);
- switch(ch) {
- case '\'':
- case '\\': writer.write('\\'); writer.write(ch); break;
- default: writer.write(ch); break;
+ if (ch=='\'' || ch=='\\') {
+ writer.write('\\');
}
+ writer.write(ch);
}
} else {
writer.write(val);
Modified: lucene/solr/trunk/src/java/org/apache/solr/request/TextResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/request/TextResponseWriter.java?rev=584574&r1=584573&r2=584574&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/request/TextResponseWriter.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/request/TextResponseWriter.java Sun Oct 14 11:38:54 2007
@@ -19,6 +19,7 @@
import org.apache.lucene.document.Document;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.FastWriter;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.DocList;
import java.io.IOException;
@@ -34,7 +35,7 @@
*/
public abstract class TextResponseWriter {
- protected final Writer writer;
+ protected final FastWriter writer;
protected final IndexSchema schema;
protected final SolrQueryRequest req;
protected final SolrQueryResponse rsp;
@@ -47,7 +48,7 @@
public TextResponseWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
- this.writer = writer;
+ this.writer = FastWriter.wrap(writer);
this.schema = req.getSchema();
this.req = req;
this.rsp = rsp;