You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2012/04/19 17:10:13 UTC
svn commit: r1327982 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/schema/ core/src/test/org/apache/solr/schema/
Author: ab
Date: Thu Apr 19 15:10:12 2012
New Revision: 1327982
URL: http://svn.apache.org/viewvc?rev=1327982&view=rev
Log:
SOLR-1535 Pre-analyzed field type.
Added:
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/JsonPreAnalyzedParser.java (with props)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java (with props)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SimplePreAnalyzedParser.java (with props)
lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java (with props)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1327982&r1=1327981&r2=1327982&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Thu Apr 19 15:10:12 2012
@@ -267,6 +267,9 @@ New Features
* SOLR-3358: Logging events are captured and available from the /admin/logging
request handler. (ryan)
+* SOLR-1535: PreAnalyzedField type provides a functionality to index (and optionally store)
+ field content that was already processed and split into tokens using some external processing
+ chain. Serialization format is pluggable, and defaults to JSON. (ab)
Optimizations
----------------------
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/JsonPreAnalyzedParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/JsonPreAnalyzedParser.java?rev=1327982&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/JsonPreAnalyzedParser.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/JsonPreAnalyzedParser.java Thu Apr 19 15:10:12 2012
@@ -0,0 +1,280 @@
+package org.apache.solr.schema;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.Payload;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.AttributeSource.State;
+import org.apache.lucene.util.BytesRef;
+import org.apache.noggit.JSONUtil;
+import org.apache.noggit.JSONWriter;
+import org.apache.noggit.ObjectBuilder;
+import org.apache.solr.common.util.Base64;
+import org.apache.solr.schema.PreAnalyzedField.ParseResult;
+import org.apache.solr.schema.PreAnalyzedField.PreAnalyzedParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class JsonPreAnalyzedParser implements PreAnalyzedParser {
+ private static final Logger LOG = LoggerFactory.getLogger(JsonPreAnalyzedParser.class);
+
+ public static final String VERSION = "1";
+
+ public static final String VERSION_KEY = "v";
+ public static final String STRING_KEY = "str";
+ public static final String BINARY_KEY = "bin";
+ public static final String TOKENS_KEY = "tokens";
+ public static final String TOKEN_KEY = "t";
+ public static final String OFFSET_START_KEY = "s";
+ public static final String OFFSET_END_KEY = "e";
+ public static final String POSINCR_KEY = "i";
+ public static final String PAYLOAD_KEY = "p";
+ public static final String TYPE_KEY = "y";
+ public static final String FLAGS_KEY = "f";
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public ParseResult parse(Reader reader, AttributeSource parent)
+ throws IOException {
+ ParseResult res = new ParseResult();
+ StringBuilder sb = new StringBuilder();
+ char[] buf = new char[128];
+ int cnt;
+ while ((cnt = reader.read(buf)) > 0) {
+ sb.append(buf, 0, cnt);
+ }
+ String val = sb.toString();
+ // empty string - accept even without version number
+ if (val.length() == 0) {
+ return res;
+ }
+ Object o = ObjectBuilder.fromJSON(val);
+ if (!(o instanceof Map)) {
+ throw new IOException("Invalid JSON type " + o.getClass().getName() + ", expected Map");
+ }
+ Map<String,Object> map = (Map<String,Object>)o;
+ // check version
+ String version = (String)map.get(VERSION_KEY);
+ if (version == null) {
+ throw new IOException("Missing VERSION key");
+ }
+ if (!VERSION.equals(version)) {
+ throw new IOException("Unknown VERSION '" + version + "', expected " + VERSION);
+ }
+ if (map.containsKey(STRING_KEY) && map.containsKey(BINARY_KEY)) {
+ throw new IOException("Field cannot have both stringValue and binaryValue");
+ }
+ res.str = (String)map.get(STRING_KEY);
+ String bin = (String)map.get(BINARY_KEY);
+ if (bin != null) {
+ byte[] data = Base64.base64ToByteArray(bin);
+ res.bin = data;
+ }
+ List<Object> tokens = (List<Object>)map.get(TOKENS_KEY);
+ if (tokens == null) {
+ return res;
+ }
+ int tokenStart = 0;
+ int tokenEnd = 0;
+ parent.clearAttributes();
+ for (Object ot : tokens) {
+ tokenStart = tokenEnd + 1; // automatic increment by 1 separator
+ Map<String,Object> tok = (Map<String,Object>)ot;
+ boolean hasOffsetStart = false;
+ boolean hasOffsetEnd = false;
+ int len = -1;
+ for (Entry<String,Object> e : tok.entrySet()) {
+ String key = e.getKey();
+ if (key.equals(TOKEN_KEY)) {
+ CharTermAttribute catt = parent.addAttribute(CharTermAttribute.class);
+ String str = String.valueOf(e.getValue());
+ catt.append(str);
+ len = str.length();
+ } else if (key.equals(OFFSET_START_KEY)) {
+ Object obj = e.getValue();
+ hasOffsetStart = true;
+ if (obj instanceof Number) {
+ tokenStart = ((Number)obj).intValue();
+ } else {
+ try {
+ tokenStart = Integer.parseInt(String.valueOf(obj));
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid " + OFFSET_START_KEY + " attribute, skipped: '" + obj + "'");
+ hasOffsetStart = false;
+ }
+ }
+ } else if (key.equals(OFFSET_END_KEY)) {
+ hasOffsetEnd = true;
+ Object obj = e.getValue();
+ if (obj instanceof Number) {
+ tokenEnd = ((Number)obj).intValue();
+ } else {
+ try {
+ tokenEnd = Integer.parseInt(String.valueOf(obj));
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid " + OFFSET_END_KEY + " attribute, skipped: '" + obj + "'");
+ hasOffsetEnd = false;
+ }
+ }
+ } else if (key.equals(POSINCR_KEY)) {
+ Object obj = e.getValue();
+ int posIncr = 1;
+ if (obj instanceof Number) {
+ posIncr = ((Number)obj).intValue();
+ } else {
+ try {
+ posIncr = Integer.parseInt(String.valueOf(obj));
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid " + POSINCR_KEY + " attribute, skipped: '" + obj + "'");
+ }
+ }
+ PositionIncrementAttribute patt = parent.addAttribute(PositionIncrementAttribute.class);
+ patt.setPositionIncrement(posIncr);
+ } else if (key.equals(PAYLOAD_KEY)) {
+ String str = String.valueOf(e.getValue());
+ if (str.length() > 0) {
+ byte[] data = Base64.base64ToByteArray(str);
+ PayloadAttribute p = parent.addAttribute(PayloadAttribute.class);
+ if (data != null && data.length > 0) {
+ p.setPayload(new Payload(data));
+ }
+ }
+ } else if (key.equals(FLAGS_KEY)) {
+ try {
+ int f = Integer.parseInt(String.valueOf(e.getValue()), 16);
+ FlagsAttribute flags = parent.addAttribute(FlagsAttribute.class);
+ flags.setFlags(f);
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid " + FLAGS_KEY + " attribute, skipped: '" + e.getValue() + "'");
+ }
+ } else if (key.equals(TYPE_KEY)) {
+ TypeAttribute tattr = parent.addAttribute(TypeAttribute.class);
+ tattr.setType(String.valueOf(e.getValue()));
+ } else {
+ LOG.warn("Unknown attribute, skipped: " + e.getKey() + "=" + e.getValue());
+ }
+ }
+ // handle offset attr
+ OffsetAttribute offset = parent.addAttribute(OffsetAttribute.class);
+ if (!hasOffsetEnd && len > -1) {
+ tokenEnd = tokenStart + len;
+ }
+ offset.setOffset(tokenStart, tokenEnd);
+ if (!hasOffsetStart) {
+ tokenStart = tokenEnd + 1;
+ }
+ // capture state and add to result
+ State state = parent.captureState();
+ res.states.add(state.clone());
+ // reset for reuse
+ parent.clearAttributes();
+ }
+ return res;
+ }
+
+ @Override
+ public String toFormattedString(Field f) throws IOException {
+ Map<String,Object> map = new HashMap<String,Object>();
+ map.put(VERSION_KEY, VERSION);
+ if (f.fieldType().stored()) {
+ String stringValue = f.stringValue();
+ if (stringValue != null) {
+ map.put(STRING_KEY, stringValue);
+ }
+ BytesRef binaryValue = f.binaryValue();
+ if (binaryValue != null) {
+ map.put(BINARY_KEY, Base64.byteArrayToBase64(binaryValue.bytes, binaryValue.offset, binaryValue.length));
+ }
+ }
+ TokenStream ts = f.tokenStreamValue();
+ if (ts != null) {
+ List<Map<String,Object>> tokens = new LinkedList<Map<String,Object>>();
+ while (ts.incrementToken()) {
+ Iterator<Class<? extends Attribute>> it = ts.getAttributeClassesIterator();
+ String cTerm = null;
+ String tTerm = null;
+ Map<String,Object> tok = new TreeMap<String,Object>();
+ while (it.hasNext()) {
+ Class<? extends Attribute> cl = it.next();
+ if (!ts.hasAttribute(cl)) {
+ continue;
+ }
+ Attribute att = ts.getAttribute(cl);
+ if (cl.isAssignableFrom(CharTermAttribute.class)) {
+ CharTermAttribute catt = (CharTermAttribute)att;
+ cTerm = new String(catt.buffer(), 0, catt.length());
+ } else if (cl.isAssignableFrom(TermToBytesRefAttribute.class)) {
+ TermToBytesRefAttribute tatt = (TermToBytesRefAttribute)att;
+ tTerm = tatt.getBytesRef().utf8ToString();
+ } else {
+ if (cl.isAssignableFrom(FlagsAttribute.class)) {
+ tok.put(FLAGS_KEY, Integer.toHexString(((FlagsAttribute)att).getFlags()));
+ } else if (cl.isAssignableFrom(OffsetAttribute.class)) {
+ tok.put(OFFSET_START_KEY, ((OffsetAttribute)att).startOffset());
+ tok.put(OFFSET_END_KEY, ((OffsetAttribute)att).endOffset());
+ } else if (cl.isAssignableFrom(PayloadAttribute.class)) {
+ Payload p = ((PayloadAttribute)att).getPayload();
+ if (p != null && p.length() > 0) {
+ tok.put(PAYLOAD_KEY, Base64.byteArrayToBase64(p.getData(), p.getOffset(), p.length()));
+ }
+ } else if (cl.isAssignableFrom(PositionIncrementAttribute.class)) {
+ tok.put(POSINCR_KEY, ((PositionIncrementAttribute)att).getPositionIncrement());
+ } else if (cl.isAssignableFrom(TypeAttribute.class)) {
+ tok.put(TYPE_KEY, ((TypeAttribute)att).type());
+ } else {
+ tok.put(cl.getName(), att.toString());
+ }
+ }
+ }
+ String term = null;
+ if (cTerm != null) {
+ term = cTerm;
+ } else {
+ term = tTerm;
+ }
+ if (term != null && term.length() > 0) {
+ tok.put(TOKEN_KEY, term);
+ }
+ tokens.add(tok);
+ }
+ map.put(TOKENS_KEY, tokens);
+ }
+ return JSONUtil.toJSON(map, -1);
+ }
+
+}
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java?rev=1327982&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java Thu Apr 19 15:10:12 2012
@@ -0,0 +1,253 @@
+package org.apache.solr.schema;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.lang.reflect.Constructor;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.AttributeSource.State;
+import org.apache.solr.analysis.SolrAnalyzer;
+import org.apache.solr.response.TextResponseWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Pre-analyzed field type provides a way to index a serialized token stream,
+ * optionally with an independent stored value of a field.
+ */
+public class PreAnalyzedField extends FieldType {
+ private static final Logger LOG = LoggerFactory.getLogger(PreAnalyzedField.class);
+
+ /** Init argument name. Value is a fully-qualified class name of the parser
+ * that implements {@link PreAnalyzedParser}.
+ */
+ public static final String PARSER_IMPL = "parserImpl";
+
+ private static final String DEFAULT_IMPL = JsonPreAnalyzedParser.class.getName();
+
+
+ private PreAnalyzedParser parser;
+
+ @Override
+ protected void init(IndexSchema schema, Map<String, String> args) {
+ super.init(schema, args);
+ String implName = args.get(PARSER_IMPL);
+ if (implName == null) {
+ parser = new JsonPreAnalyzedParser();
+ } else {
+ try {
+ Class<?> implClazz = Class.forName(implName);
+ if (!PreAnalyzedParser.class.isAssignableFrom(implClazz)) {
+ throw new Exception("must implement " + PreAnalyzedParser.class.getName());
+ }
+ Constructor<?> c = implClazz.getConstructor(new Class<?>[0]);
+ parser = (PreAnalyzedParser) c.newInstance(new Object[0]);
+ } catch (Exception e) {
+ LOG.warn("Can't use the configured PreAnalyzedParser class '" + implName + "' (" +
+ e.getMessage() + "), using default " + DEFAULT_IMPL);
+ parser = new JsonPreAnalyzedParser();
+ }
+ }
+ }
+
+ @Override
+ public Analyzer getAnalyzer() {
+ return new SolrAnalyzer() {
+
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName,
+ Reader reader) {
+ try {
+ return new TokenStreamComponents(new PreAnalyzedTokenizer(reader, parser));
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ };
+ }
+
+ @Override
+ public Analyzer getQueryAnalyzer() {
+ return getAnalyzer();
+ }
+
+ @Override
+ public IndexableField createField(SchemaField field, Object value,
+ float boost) {
+ IndexableField f = null;
+ try {
+ f = fromString(field, String.valueOf(value), boost);
+ } catch (Exception e) {
+ e.printStackTrace();
+ return null;
+ }
+ return f;
+ }
+
+ @Override
+ public SortField getSortField(SchemaField field, boolean top) {
+ return getStringSort(field, top);
+ }
+
+ @Override
+ public void write(TextResponseWriter writer, String name, IndexableField f)
+ throws IOException {
+ writer.writeStr(name, f.stringValue(), true);
+ }
+
+ /** Utility method to convert a field to a string that is parse-able by this
+ * class.
+ * @param f field to convert
+ * @return string that is compatible with the serialization format
+ * @throws IOException
+ */
+ public String toFormattedString(Field f) throws IOException {
+ return parser.toFormattedString(f);
+ }
+
+ /**
+ * This is a simple holder of a stored part and the collected states (tokens with attributes).
+ */
+ public static class ParseResult {
+ public String str;
+ public byte[] bin;
+ public List<State> states = new LinkedList<State>();
+ }
+
+ /**
+ * Parse the input and return the stored part and the tokens with attributes.
+ */
+ public static interface PreAnalyzedParser {
+ /**
+ * Parse input.
+ * @param reader input to read from
+ * @param parent parent who will own the resulting states (tokens with attributes)
+ * @return parse result, with possibly null stored and/or states fields.
+ * @throws IOException if a parsing error or IO error occurs
+ */
+ public ParseResult parse(Reader reader, AttributeSource parent) throws IOException;
+
+ /**
+ * Format a field so that the resulting String is valid for parsing with {@link #parse(Reader, AttributeSource)}.
+ * @param f field instance
+ * @return formatted string
+ * @throws IOException
+ */
+ public String toFormattedString(Field f) throws IOException;
+ }
+
+
+ public IndexableField fromString(SchemaField field, String val, float boost) throws Exception {
+ if (val == null || val.trim().length() == 0) {
+ return null;
+ }
+ PreAnalyzedTokenizer parse = new PreAnalyzedTokenizer(new StringReader(val), parser);
+ Field f = (Field)super.createField(field, val, boost);
+ if (parse.getStringValue() != null) {
+ f.setStringValue(parse.getStringValue());
+ } else if (parse.getBinaryValue() != null) {
+ f.setBytesValue(parse.getBinaryValue());
+ } else {
+ f.fieldType().setStored(false);
+ }
+
+ if (parse.hasTokenStream()) {
+ f.fieldType().setIndexed(true);
+ f.fieldType().setTokenized(true);
+ f.setTokenStream(parse);
+ }
+ return f;
+ }
+
+ /**
+ * Token stream that works from a list of saved states.
+ */
+ private static class PreAnalyzedTokenizer extends Tokenizer {
+ private final List<AttributeSource.State> cachedStates = new LinkedList<AttributeSource.State>();
+ private Iterator<AttributeSource.State> it = null;
+ private String stringValue = null;
+ private byte[] binaryValue = null;
+ private PreAnalyzedParser parser;
+
+ public PreAnalyzedTokenizer(Reader reader, PreAnalyzedParser parser) throws IOException {
+ super(reader);
+ this.parser = parser;
+ reset(reader);
+ }
+
+ public boolean hasTokenStream() {
+ return !cachedStates.isEmpty();
+ }
+
+ public String getStringValue() {
+ return stringValue;
+ }
+
+ public byte[] getBinaryValue() {
+ return binaryValue;
+ }
+
+ public final boolean incrementToken() throws IOException {
+ // lazy init the iterator
+ if (it == null) {
+ it = cachedStates.iterator();
+ }
+
+ if (!it.hasNext()) {
+ return false;
+ }
+
+ AttributeSource.State state = (State) it.next();
+ restoreState(state.clone());
+ return true;
+ }
+
+ public final void reset() {
+ it = cachedStates.iterator();
+ }
+
+ @Override
+ public void reset(Reader input) throws IOException {
+ super.reset(input);
+ cachedStates.clear();
+ stringValue = null;
+ binaryValue = null;
+ ParseResult res = parser.parse(input, this);
+ if (res != null) {
+ stringValue = res.str;
+ binaryValue = res.bin;
+ if (res.states != null) {
+ cachedStates.addAll(res.states);
+ }
+ }
+ }
+ }
+
+}
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SimplePreAnalyzedParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SimplePreAnalyzedParser.java?rev=1327982&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SimplePreAnalyzedParser.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SimplePreAnalyzedParser.java Thu Apr 19 15:10:12 2012
@@ -0,0 +1,573 @@
+package org.apache.solr.schema;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.Payload;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.AttributeSource.State;
+import org.apache.solr.schema.PreAnalyzedField.ParseResult;
+import org.apache.solr.schema.PreAnalyzedField.PreAnalyzedParser;
+
+/**
+ * Simple plain text format parser for {@link PreAnalyzedField}.
+ * <h2>Serialization format</h2>
+ * <p>The format of the serialization is as follows:
+ * <pre>
+ * content ::= version (stored)? tokens
+ * version ::= digit+ " "
+ * ; stored field value - any "=" inside must be escaped!
+ * stored ::= "=" text "="
+ * tokens ::= (token ((" ") + token)*)*
+ * token ::= text ("," attrib)*
+ * attrib ::= name '=' value
+ * name ::= text
+ * value ::= text
+ * </pre>
+ * <p>Special characters in "text" values can be escaped
+ * using the escape character \ . The following escape sequences are recognized:
+ * <pre>
+ * "\ " - literal space character
+ * "\," - literal , character
+ * "\=" - literal = character
+ * "\\" - literal \ character
+ * "\n" - newline
+ * "\r" - carriage return
+ * "\t" - horizontal tab
+ * </pre>
+ * Please note that Unicode sequences (e.g. \u0001) are not supported.
+ * <h2>Supported attribute names</h2>
+ * The following token attributes are supported, and identified with short
+ * symbolic names:
+ * <pre>
+ * i - position increment (integer)
+ * s - token offset, start position (integer)
+ * e - token offset, end position (integer)
+ * t - token type (string)
+ * f - token flags (hexadecimal integer)
+ * p - payload (bytes in hexadecimal format)
+ * </pre>
+ * Token positions are tracked and implicitly added to the token stream -
+ * the start and end offsets consider only the term text and whitespace,
+ * and exclude the space taken by token attributes.
+ * <h2>Example token streams</h2>
+ * <pre>
+ * 1 one two three
+ - version 1
+ - stored: 'null'
+ - tok: '(term=one,startOffset=0,endOffset=3)'
+ - tok: '(term=two,startOffset=4,endOffset=7)'
+ - tok: '(term=three,startOffset=8,endOffset=13)'
+ 1 one two three
+ - version 1
+ - stored: 'null'
+ - tok: '(term=one,startOffset=1,endOffset=4)'
+ - tok: '(term=two,startOffset=6,endOffset=9)'
+ - tok: '(term=three,startOffset=12,endOffset=17)'
+1 one,s=123,e=128,i=22 two three,s=20,e=22
+ - version 1
+ - stored: 'null'
+ - tok: '(term=one,positionIncrement=22,startOffset=123,endOffset=128)'
+ - tok: '(term=two,positionIncrement=1,startOffset=5,endOffset=8)'
+ - tok: '(term=three,positionIncrement=1,startOffset=20,endOffset=22)'
+1 \ one\ \,,i=22,a=\, two\=
+
+ \n,\ =\ \
+ - version 1
+ - stored: 'null'
+ - tok: '(term= one ,,positionIncrement=22,startOffset=0,endOffset=6)'
+ - tok: '(term=two=
+
+
+ ,positionIncrement=1,startOffset=7,endOffset=15)'
+ - tok: '(term=\,positionIncrement=1,startOffset=17,endOffset=18)'
+1 ,i=22 ,i=33,s=2,e=20 ,
+ - version 1
+ - stored: 'null'
+ - tok: '(term=,positionIncrement=22,startOffset=0,endOffset=0)'
+ - tok: '(term=,positionIncrement=33,startOffset=2,endOffset=20)'
+ - tok: '(term=,positionIncrement=1,startOffset=2,endOffset=2)'
+1 =This is the stored part with \=
+ \n \t escapes.=one two three
+ - version 1
+ - stored: 'This is the stored part with =
+ \n \t escapes.'
+ - tok: '(term=one,startOffset=0,endOffset=3)'
+ - tok: '(term=two,startOffset=4,endOffset=7)'
+ - tok: '(term=three,startOffset=8,endOffset=13)'
+1 ==
+ - version 1
+ - stored: ''
+ - (no tokens)
+1 =this is a test.=
+ - version 1
+ - stored: 'this is a test.'
+ - (no tokens)
+ * </pre>
+ */
+public final class SimplePreAnalyzedParser implements PreAnalyzedParser {
+ static final String VERSION = "1";
+
+ private static class Tok {
+ StringBuilder token = new StringBuilder();
+ Map<String, String> attr = new HashMap<String, String>();
+
+ public boolean isEmpty() {
+ return token.length() == 0 && attr.size() == 0;
+ }
+
+ public void reset() {
+ token.setLength(0);
+ attr.clear();
+ }
+
+ public String toString() {
+ return "tok='" + token + "',attr=" + attr;
+ }
+ }
+
+ // parser state
+ private static enum S {TOKEN, NAME, VALUE, UNDEF};
+
+ private static final byte[] EMPTY_BYTES = new byte[0];
+
+ /** Utility method to convert byte array to a hex string. */
+ static byte[] hexToBytes(String hex) {
+ if (hex == null) {
+ return EMPTY_BYTES;
+ }
+ hex = hex.replaceAll("\\s+", "");
+ if (hex.length() == 0) {
+ return EMPTY_BYTES;
+ }
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(hex.length() / 2);
+ byte b;
+ for (int i = 0; i < hex.length(); i++) {
+ int high = charToNibble(hex.charAt(i));
+ int low = 0;
+ if (i < hex.length() - 1) {
+ i++;
+ low = charToNibble(hex.charAt(i));
+ }
+ b = (byte)(high << 4 | low);
+ baos.write(b);
+ }
+ return baos.toByteArray();
+ }
+
+ static final int charToNibble(char c) {
+ if (c >= '0' && c <= '9') {
+ return c - '0';
+ } else if (c >= 'a' && c <= 'f') {
+ return 0xa + (c - 'a');
+ } else if (c >= 'A' && c <= 'F') {
+ return 0xA + (c - 'A');
+ } else {
+ throw new RuntimeException("Not a hex character: '" + c + "'");
+ }
+ }
+
+ static String bytesToHex(byte bytes[], int offset, int length) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = offset; i < offset + length; ++i) {
+ sb.append(Integer.toHexString(0x0100 + (bytes[i] & 0x00FF))
+ .substring(1));
+ }
+ return sb.toString();
+ }
+
+ public SimplePreAnalyzedParser() {
+
+ }
+
+ @Override
+ public ParseResult parse(Reader reader, AttributeSource parent) throws IOException {
+ ParseResult res = new ParseResult();
+ StringBuilder sb = new StringBuilder();
+ char[] buf = new char[128];
+ int cnt;
+ while ((cnt = reader.read(buf)) > 0) {
+ sb.append(buf, 0, cnt);
+ }
+ String val = sb.toString();
+ // empty string - accept even without version number
+ if (val.length() == 0) {
+ return res;
+ }
+ // first consume the version
+ int idx = val.indexOf(' ');
+ if (idx == -1) {
+ throw new IOException("Missing VERSION token");
+ }
+ String version = val.substring(0, idx);
+ if (!VERSION.equals(version)) {
+ throw new IOException("Unknown VERSION " + version);
+ }
+ val = val.substring(idx + 1);
+ // then consume the optional stored part
+ int tsStart = 0;
+ boolean hasStored = false;
+ StringBuilder storedBuf = new StringBuilder();
+ if (val.charAt(0) == '=') {
+ hasStored = true;
+ if (val.length() > 1) {
+ for (int i = 1; i < val.length(); i++) {
+ char c = val.charAt(i);
+ if (c == '\\') {
+ if (i < val.length() - 1) {
+ c = val.charAt(++i);
+ if (c == '=') { // we recognize only \= escape in the stored part
+ storedBuf.append('=');
+ } else {
+ storedBuf.append('\\');
+ storedBuf.append(c);
+ continue;
+ }
+ } else {
+ storedBuf.append(c);
+ continue;
+ }
+ } else if (c == '=') {
+ // end of stored text
+ tsStart = i + 1;
+ break;
+ } else {
+ storedBuf.append(c);
+ }
+ }
+ if (tsStart == 0) { // missing end-of-stored marker
+ throw new IOException("Missing end marker of stored part");
+ }
+ } else {
+ throw new IOException("Unexpected end of stored field");
+ }
+ }
+ if (hasStored) {
+ res.str = storedBuf.toString();
+ }
+ Tok tok = new Tok();
+ StringBuilder attName = new StringBuilder();
+ StringBuilder attVal = new StringBuilder();
+ // parser state
+ S s = S.UNDEF;
+ int lastPos = 0;
+ for (int i = tsStart; i < val.length(); i++) {
+ char c = val.charAt(i);
+ if (c == ' ') {
+ // collect leftovers
+ switch (s) {
+ case VALUE :
+ if (attVal.length() == 0) {
+ throw new IOException("Unexpected character '" + c + "' at position " + i + " - empty value of attribute.");
+ }
+ if (attName.length() > 0) {
+ tok.attr.put(attName.toString(), attVal.toString());
+ }
+ break;
+ case NAME: // attr name without a value ?
+ if (attName.length() > 0) {
+ throw new IOException("Unexpected character '" + c + "' at position " + i + " - missing attribute value.");
+ } else {
+ // accept missing att name and value
+ }
+ break;
+ case TOKEN:
+ case UNDEF:
+ // do nothing, advance to next token
+ }
+ attName.setLength(0);
+ attVal.setLength(0);
+ if (!tok.isEmpty() || s == S.NAME) {
+ AttributeSource.State state = createState(parent, tok, lastPos);
+ if (state != null) res.states.add(state.clone());
+ }
+ // reset tok
+ s = S.UNDEF;
+ tok.reset();
+ // skip
+ lastPos++;
+ continue;
+ }
+ StringBuilder tgt = null;
+ switch (s) {
+ case TOKEN:
+ tgt = tok.token;
+ break;
+ case NAME:
+ tgt = attName;
+ break;
+ case VALUE:
+ tgt = attVal;
+ break;
+ case UNDEF:
+ tgt = tok.token;
+ s = S.TOKEN;
+ }
+ if (c == '\\') {
+ if (s == S.TOKEN) lastPos++;
+ if (i >= val.length() - 1) { // end
+
+ tgt.append(c);
+ continue;
+ } else {
+ c = val.charAt(++i);
+ switch (c) {
+ case '\\' :
+ case '=' :
+ case ',' :
+ case ' ' :
+ tgt.append(c);
+ break;
+ case 'n':
+ tgt.append('\n');
+ break;
+ case 'r':
+ tgt.append('\r');
+ break;
+ case 't':
+ tgt.append('\t');
+ break;
+ default:
+ tgt.append('\\');
+ tgt.append(c);
+ lastPos++;
+ }
+ }
+ } else {
+ // state switch
+ if (c == ',') {
+ if (s == S.TOKEN) {
+ s = S.NAME;
+ } else if (s == S.VALUE) { // end of value, start of next attr
+ if (attVal.length() == 0) {
+ throw new IOException("Unexpected character '" + c + "' at position " + i + " - empty value of attribute.");
+ }
+ if (attName.length() > 0 && attVal.length() > 0) {
+ tok.attr.put(attName.toString(), attVal.toString());
+ }
+ // reset
+ attName.setLength(0);
+ attVal.setLength(0);
+ s = S.NAME;
+ } else {
+ throw new IOException("Unexpected character '" + c + "' at position " + i + " - missing attribute value.");
+ }
+ } else if (c == '=') {
+ if (s == S.NAME) {
+ s = S.VALUE;
+ } else {
+ throw new IOException("Unexpected character '" + c + "' at position " + i + " - empty value of attribute.");
+ }
+ } else {
+ tgt.append(c);
+ if (s == S.TOKEN) lastPos++;
+ }
+ }
+ }
+ // collect leftovers
+ if (!tok.isEmpty() || s == S.NAME || s == S.VALUE) {
+ // remaining attrib?
+ if (s == S.VALUE) {
+ if (attName.length() > 0 && attVal.length() > 0) {
+ tok.attr.put(attName.toString(), attVal.toString());
+ }
+ }
+ AttributeSource.State state = createState(parent, tok, lastPos);
+ if (state != null) res.states.add(state.clone());
+ }
+ return res;
+ }
+
+ private static AttributeSource.State createState(AttributeSource a, Tok state, int tokenEnd) {
+ a.clearAttributes();
+ CharTermAttribute termAtt = a.addAttribute(CharTermAttribute.class);
+ char[] tokChars = state.token.toString().toCharArray();
+ termAtt.copyBuffer(tokChars, 0, tokChars.length);
+ int tokenStart = tokenEnd - state.token.length();
+ for (Entry<String, String> e : state.attr.entrySet()) {
+ String k = e.getKey();
+ if (k.equals("i")) {
+ // position increment
+ int incr = Integer.parseInt(e.getValue());
+ PositionIncrementAttribute posIncr = a.addAttribute(PositionIncrementAttribute.class);
+ posIncr.setPositionIncrement(incr);
+ } else if (k.equals("s")) {
+ tokenStart = Integer.parseInt(e.getValue());
+ } else if (k.equals("e")) {
+ tokenEnd = Integer.parseInt(e.getValue());
+ } else if (k.equals("y")) {
+ TypeAttribute type = a.addAttribute(TypeAttribute.class);
+ type.setType(e.getValue());
+ } else if (k.equals("f")) {
+ FlagsAttribute flags = a.addAttribute(FlagsAttribute.class);
+ int f = Integer.parseInt(e.getValue(), 16);
+ flags.setFlags(f);
+ } else if (k.equals("p")) {
+ PayloadAttribute p = a.addAttribute(PayloadAttribute.class);
+ byte[] data = hexToBytes(e.getValue());
+ if (data != null && data.length > 0) {
+ p.setPayload(new Payload(data));
+ }
+ } else {
+ // unknown attribute
+ }
+ }
+ // handle offset attr
+ OffsetAttribute offset = a.addAttribute(OffsetAttribute.class);
+ offset.setOffset(tokenStart, tokenEnd);
+ State resState = a.captureState();
+ a.clearAttributes();
+ return resState;
+ }
+
+ public String toFormattedString(Field f) throws IOException {
+ StringBuilder sb = new StringBuilder();
+ sb.append(VERSION + " ");
+ if (f.fieldType().stored()) {
+ String s = f.stringValue();
+ if (s != null) {
+ // encode the equals sign
+ s = s.replaceAll("=", "\\=");
+ sb.append('=');
+ sb.append(s);
+ sb.append('=');
+ }
+ }
+ TokenStream ts = f.tokenStreamValue();
+ if (ts != null) {
+ StringBuilder tok = new StringBuilder();
+ boolean next = false;
+ while (ts.incrementToken()) {
+ if (next) {
+ sb.append(' ');
+ } else {
+ next = true;
+ }
+ tok.setLength(0);
+ Iterator<Class<? extends Attribute>> it = ts.getAttributeClassesIterator();
+ String cTerm = null;
+ String tTerm = null;
+ while (it.hasNext()) {
+ Class<? extends Attribute> cl = it.next();
+ if (!ts.hasAttribute(cl)) {
+ continue;
+ }
+ Attribute att = ts.getAttribute(cl);
+ if (cl.isAssignableFrom(CharTermAttribute.class)) {
+ CharTermAttribute catt = (CharTermAttribute)att;
+ cTerm = escape(catt.buffer(), catt.length());
+ } else if (cl.isAssignableFrom(TermToBytesRefAttribute.class)) {
+ TermToBytesRefAttribute tatt = (TermToBytesRefAttribute)att;
+ char[] tTermChars = tatt.getBytesRef().utf8ToString().toCharArray();
+ tTerm = escape(tTermChars, tTermChars.length);
+ } else {
+ if (tok.length() > 0) tok.append(',');
+ if (cl.isAssignableFrom(FlagsAttribute.class)) {
+ tok.append("f=" + Integer.toHexString(((FlagsAttribute)att).getFlags()));
+ } else if (cl.isAssignableFrom(OffsetAttribute.class)) {
+ tok.append("s=" + ((OffsetAttribute)att).startOffset() + ",e=" + ((OffsetAttribute)att).endOffset());
+ } else if (cl.isAssignableFrom(PayloadAttribute.class)) {
+ Payload p = ((PayloadAttribute)att).getPayload();
+ if (p != null && p.length() > 0) {
+ tok.append("p=" + bytesToHex(p.getData(), p.getOffset(), p.length()));
+ } else if (tok.length() > 0) {
+ tok.setLength(tok.length() - 1); // remove the last comma
+ }
+ } else if (cl.isAssignableFrom(PositionIncrementAttribute.class)) {
+ tok.append("i=" + ((PositionIncrementAttribute)att).getPositionIncrement());
+ } else if (cl.isAssignableFrom(TypeAttribute.class)) {
+ tok.append("y=" + escape(((TypeAttribute)att).type()));
+ } else {
+
+ tok.append(cl.getName() + "=" + escape(att.toString()));
+ }
+ }
+ }
+ String term = null;
+ if (cTerm != null) {
+ term = cTerm;
+ } else {
+ term = tTerm;
+ }
+ if (term != null && term.length() > 0) {
+ if (tok.length() > 0) {
+ tok.insert(0, term + ",");
+ } else {
+ tok.insert(0, term);
+ }
+ }
+ sb.append(tok);
+ }
+ }
+ return sb.toString();
+ }
+
+ String escape(String val) {
+ return escape(val.toCharArray(), val.length());
+ }
+
+ String escape(char[] val, int len) {
+ if (val == null || len == 0) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < len; i++) {
+ switch (val[i]) {
+ case '\\' :
+ case '=' :
+ case ',' :
+ case ' ' :
+ sb.append('\\');
+ sb.append(val[i]);
+ break;
+ case '\n' :
+ sb.append('\\');
+ sb.append('n');
+ break;
+ case '\r' :
+ sb.append('\\');
+ sb.append('r');
+ break;
+ case '\t' :
+ sb.append('\\');
+ sb.append('t');
+ break;
+ default:
+ sb.append(val[i]);
+ }
+ }
+ return sb.toString();
+ }
+
+}
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java?rev=1327982&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java Thu Apr 19 15:10:12 2012
@@ -0,0 +1,150 @@
+package org.apache.solr.schema;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Collections;
+import java.util.HashMap;
+
+import org.apache.lucene.document.Field;
+import org.apache.solr.common.util.Base64;
+import org.apache.solr.schema.PreAnalyzedField.PreAnalyzedParser;
+import org.junit.Test;
+
+import junit.framework.TestCase;
+
+public class PreAnalyzedFieldTest extends TestCase {
+
+ private static final String[] valid = {
+ "1 one two three", // simple parsing
+ "1 one two three ", // spurious spaces
+ "1 one,s=123,e=128,i=22 two three,s=20,e=22,y=foobar", // attribs
+ "1 \\ one\\ \\,,i=22,a=\\, two\\=\n\r\t\\n,\\ =\\ \\", // escape madness
+ "1 ,i=22 ,i=33,s=2,e=20 , ", // empty token text, non-empty attribs
+ "1 =This is the stored part with \\= \n \\n \t \\t escapes.=one two three \u0001Ä
ÄÄÅÅóÅźż", // stored plus token stream
+ "1 ==", // empty stored, no token stream
+ "1 =this is a test.=", // stored + empty token stream
+ "1 one,p=deadbeef two,p=0123456789abcdef three" // payloads
+ };
+
+ private static final String[] validParsed = {
+ "1 one,s=0,e=3 two,s=4,e=7 three,s=8,e=13",
+ "1 one,s=1,e=4 two,s=6,e=9 three,s=12,e=17",
+ "1 one,i=22,s=123,e=128,y=word two,i=1,s=5,e=8,y=word three,i=1,s=20,e=22,y=foobar",
+ "1 \\ one\\ \\,,i=22,s=0,e=6 two\\=\\n\\r\\t\\n,i=1,s=7,e=15 \\\\,i=1,s=17,e=18",
+ "1 i=22,s=0,e=0 i=33,s=2,e=20 i=1,s=2,e=2",
+ "1 =This is the stored part with = \n \\n \t \\t escapes.=one,s=0,e=3 two,s=4,e=7 three,s=8,e=13 \u0001Ä
ÄÄÅÅóÅźż,s=15,e=25",
+ "1 ==",
+ "1 =this is a test.=",
+ "1 one,p=deadbeef,s=0,e=3 two,p=0123456789abcdef,s=4,e=7 three,s=8,e=13"
+ };
+
+ private static final String[] invalid = {
+ "one two three", // missing version #
+ "2 one two three", // invalid version #
+ "1 o,ne two", // missing escape
+ "1 one t=wo", // missing escape
+ "1 one,, two", // missing attribs, unescaped comma
+ "1 one,s ", // missing attrib value
+ "1 one,s= val", // missing attrib value, unescaped space
+ "1 one,s=,val", // unescaped comma
+ "1 =", // unescaped equals
+ "1 =stored ", // unterminated stored
+ "1 ===" // empty stored (ok), but unescaped = in token stream
+ };
+
+ SchemaField field = null;
+ int props =
+ FieldProperties.INDEXED | FieldProperties.STORED;
+
+ public void setUp() {
+ field = new SchemaField("content", new TextField(), props, null);
+ }
+
+ @Test
+ public void testValidSimple() {
+ PreAnalyzedField paf = new PreAnalyzedField();
+ // use Simple format
+ HashMap<String,String> args = new HashMap<String,String>();
+ args.put(PreAnalyzedField.PARSER_IMPL, SimplePreAnalyzedParser.class.getName());
+ paf.init((IndexSchema)null, args);
+ PreAnalyzedParser parser = new SimplePreAnalyzedParser();
+ for (int i = 0; i < valid.length; i++) {
+ String s = valid[i];
+ try {
+ Field f = (Field)paf.fromString(field, s, 1.0f);
+ //System.out.println(" - toString: '" + sb.toString() + "'");
+ assertEquals(validParsed[i], parser.toFormattedString(f));
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail("Should pass: '" + s + "', exception: " + e);
+ }
+ }
+ }
+
+ @Test
+ public void testInvalidSimple() {
+ PreAnalyzedField paf = new PreAnalyzedField();
+ paf.init((IndexSchema)null, Collections.<String,String>emptyMap());
+ for (String s : invalid) {
+ try {
+ paf.fromString(field, s, 1.0f);
+ fail("should fail: '" + s + "'");
+ } catch (Exception e) {
+ //
+ }
+ }
+ }
+
+ // "1 =test Ä
ÄÄÅÅóÅźż \u0001=one,i=22,s=123,e=128,p=deadbeef,y=word two,i=1,s=5,e=8,y=word three,i=1,s=20,e=22,y=foobar"
+
+ private static final String jsonValid = "{\"v\":\"1\",\"str\":\"test Ä
ÄÄÅÅóÅźż\",\"tokens\":[" +
+ "{\"e\":128,\"i\":22,\"p\":\"DQ4KDQsODg8=\",\"s\":123,\"t\":\"one\",\"y\":\"word\"}," +
+ "{\"e\":8,\"i\":1,\"s\":5,\"t\":\"two\",\"y\":\"word\"}," +
+ "{\"e\":22,\"i\":1,\"s\":20,\"t\":\"three\",\"y\":\"foobar\"}" +
+ "]}";
+
+ @Test
+ public void testParsers() {
+ PreAnalyzedField paf = new PreAnalyzedField();
+ // use Simple format
+ HashMap<String,String> args = new HashMap<String,String>();
+ args.put(PreAnalyzedField.PARSER_IMPL, SimplePreAnalyzedParser.class.getName());
+ paf.init((IndexSchema)null, args);
+ try {
+ Field f = (Field)paf.fromString(field, valid[0], 1.0f);
+ } catch (Exception e) {
+ fail("Should pass: '" + valid[0] + "', exception: " + e);
+ }
+ // use JSON format
+ args.put(PreAnalyzedField.PARSER_IMPL, JsonPreAnalyzedParser.class.getName());
+ paf.init((IndexSchema)null, args);
+ try {
+ Field f = (Field)paf.fromString(field, valid[0], 1.0f);
+ fail("Should fail JSON parsing: '" + valid[0]);
+ } catch (Exception e) {
+ }
+ byte[] deadbeef = new byte[]{(byte)0xd, (byte)0xe, (byte)0xa, (byte)0xd, (byte)0xb, (byte)0xe, (byte)0xe, (byte)0xf};
+ PreAnalyzedParser parser = new JsonPreAnalyzedParser();
+ try {
+ Field f = (Field)paf.fromString(field, jsonValid, 1.0f);
+ assertEquals(jsonValid, parser.toFormattedString(f));
+ } catch (Exception e) {
+ fail("Should pass: '" + jsonValid + "', exception: " + e);
+ }
+ }
+}