You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2010/06/06 00:36:06 UTC
svn commit: r951785 - in /lucene/dev/trunk/lucene: CHANGES.txt
src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java
src/test/org/apache/lucene/TestExternalCodecs.java
Author: mikemccand
Date: Sat Jun 5 22:36:06 2010
New Revision: 951785
URL: http://svn.apache.org/viewvc?rev=951785&view=rev
Log:
LUCENE-2489: move PerFieldCodecWrapper to core
Added:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java (with props)
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=951785&r1=951784&r2=951785&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Sat Jun 5 22:36:06 2010
@@ -189,6 +189,8 @@ New features
* LUCENE-2440: Add support for custom ExecutorService in
ParallelMultiSearcher (Edward Drapkin via Mike McCandless)
+* LUCENE-2489: Added PerFieldCodecWrapper (in oal.index.codecs) which
+ lets you set the Codec per field (Mike McCandless)
======================= Lucene 3.x (not yet released) =======================
Added: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java?rev=951785&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java (added)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java Sat Jun 5 22:36:06 2010
@@ -0,0 +1,238 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.IdentityHashMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.io.IOException;
+
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.store.Directory;
+
+
+/** Simple Codec that dispatches field-specific codecs.
+ * You must ensure every field you index has a Codec, or
+ * the defaultCodec is non null. Also, the separate
+ * codecs cannot conflict on file names.
+ *
+ * @lucene.experimental */
+public class PerFieldCodecWrapper extends Codec {
+ private final Map<String,Codec> fields = new IdentityHashMap<String,Codec>();
+ private final Codec defaultCodec;
+
+ public PerFieldCodecWrapper(Codec defaultCodec) {
+ name = "PerField";
+ this.defaultCodec = defaultCodec;
+ }
+
+ public void add(String field, Codec codec) {
+ fields.put(field, codec);
+ }
+
+ public Codec getCodec(String field) {
+ Codec codec = fields.get(field);
+ if (codec != null) {
+ return codec;
+ } else {
+ return defaultCodec;
+ }
+ }
+
+ @Override
+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+ return new FieldsWriter(state);
+ }
+
+ private class FieldsWriter extends FieldsConsumer {
+ private final SegmentWriteState state;
+ private final Map<Codec,FieldsConsumer> codecs = new HashMap<Codec,FieldsConsumer>();
+ private final Set<String> fieldsSeen = new TreeSet<String>();
+
+ public FieldsWriter(SegmentWriteState state) {
+ this.state = state;
+ }
+
+ @Override
+ public TermsConsumer addField(FieldInfo field) throws IOException {
+ fieldsSeen.add(field.name);
+ Codec codec = getCodec(field.name);
+
+ FieldsConsumer fields = codecs.get(codec);
+ if (fields == null) {
+ fields = codec.fieldsConsumer(state);
+ codecs.put(codec, fields);
+ }
+ return fields.addField(field);
+ }
+
+ @Override
+ public void close() throws IOException {
+ Iterator<FieldsConsumer> it = codecs.values().iterator();
+ IOException err = null;
+ while(it.hasNext()) {
+ try {
+ it.next().close();
+ } catch (IOException ioe) {
+ // keep first IOException we hit but keep
+ // closing the rest
+ if (err == null) {
+ err = ioe;
+ }
+ }
+ }
+ if (err != null) {
+ throw err;
+ }
+ }
+ }
+
+ private class FieldsReader extends FieldsProducer {
+
+ private final Set<String> fields = new TreeSet<String>();
+ private final Map<Codec,FieldsProducer> codecs = new HashMap<Codec,FieldsProducer>();
+
+ public FieldsReader(Directory dir, FieldInfos fieldInfos,
+ SegmentInfo si, int readBufferSize,
+ int indexDivisor) throws IOException {
+
+ final int fieldCount = fieldInfos.size();
+ for(int i=0;i<fieldCount;i++) {
+ FieldInfo fi = fieldInfos.fieldInfo(i);
+ if (fi.isIndexed) {
+ fields.add(fi.name);
+ Codec codec = getCodec(fi.name);
+ if (!codecs.containsKey(codec)) {
+ codecs.put(codec, codec.fieldsProducer(new SegmentReadState(dir, si, fieldInfos, readBufferSize, indexDivisor)));
+ }
+ }
+ }
+ }
+
+ private final class FieldsIterator extends FieldsEnum {
+ private final Iterator<String> it;
+ private String current;
+
+ public FieldsIterator() {
+ it = fields.iterator();
+ }
+
+ @Override
+ public String next() {
+ if (it.hasNext()) {
+ current = it.next();
+ } else {
+ current = null;
+ }
+
+ return current;
+ }
+
+ @Override
+ public TermsEnum terms() throws IOException {
+ Terms terms = codecs.get(getCodec(current)).terms(current);
+ if (terms != null) {
+ return terms.iterator();
+ } else {
+ return null;
+ }
+ }
+ }
+
+ @Override
+ public FieldsEnum iterator() throws IOException {
+ return new FieldsIterator();
+ }
+
+ @Override
+ public Terms terms(String field) throws IOException {
+ Codec codec = getCodec(field);
+
+ FieldsProducer fields = codecs.get(codec);
+ assert fields != null;
+ return fields.terms(field);
+ }
+
+ @Override
+ public void close() throws IOException {
+ Iterator<FieldsProducer> it = codecs.values().iterator();
+ IOException err = null;
+ while(it.hasNext()) {
+ try {
+ it.next().close();
+ } catch (IOException ioe) {
+ // keep first IOException we hit but keep
+ // closing the rest
+ if (err == null) {
+ err = ioe;
+ }
+ }
+ }
+ if (err != null) {
+ throw err;
+ }
+ }
+
+ @Override
+ public void loadTermsIndex(int indexDivisor) throws IOException {
+ Iterator<FieldsProducer> it = codecs.values().iterator();
+ while(it.hasNext()) {
+ it.next().loadTermsIndex(indexDivisor);
+ }
+ }
+ }
+
+ public FieldsProducer fieldsProducer(SegmentReadState state)
+ throws IOException {
+ return new FieldsReader(state.dir, state.fieldInfos, state.segmentInfo, state.readBufferSize, state.termsIndexDivisor);
+ }
+
+ @Override
+ public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
+ Iterator<Codec> it = fields.values().iterator();
+ Set<Codec> seen = new HashSet<Codec>();
+ while(it.hasNext()) {
+ final Codec codec = it.next();
+ if (!seen.contains(codec)) {
+ seen.add(codec);
+ codec.files(dir, info, files);
+ }
+ }
+ }
+
+ @Override
+ public void getExtensions(Set<String> extensions) {
+ Iterator<Codec> it = fields.values().iterator();
+ while(it.hasNext()) {
+ final Codec codec = it.next();
+ codec.getExtensions(extensions);
+ }
+ }
+}
Propchange: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestExternalCodecs.java?rev=951785&r1=951784&r2=951785&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestExternalCodecs.java Sat Jun 5 22:36:06 2010
@@ -469,204 +469,6 @@ public class TestExternalCodecs extends
}
}
- /** Simple Codec that dispatches field-specific codecs.
- * You must ensure every field you index has a Codec, or
- * the defaultCodec is non null. Also, the separate
- * codecs cannot conflict on file names.*/
- // TODO: promote to core
- public static class PerFieldCodecWrapper extends Codec {
- private final Map<String,Codec> fields = new IdentityHashMap<String,Codec>();
- private final Codec defaultCodec;
-
- public PerFieldCodecWrapper(Codec defaultCodec) {
- name = "PerField";
- this.defaultCodec = defaultCodec;
- }
-
- public void add(String field, Codec codec) {
- fields.put(field, codec);
- }
-
- Codec getCodec(String field) {
- Codec codec = fields.get(field);
- if (codec != null) {
- return codec;
- } else {
- return defaultCodec;
- }
- }
-
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- return new FieldsWriter(state);
- }
-
- private class FieldsWriter extends FieldsConsumer {
- private final SegmentWriteState state;
- private final Map<Codec,FieldsConsumer> codecs = new HashMap<Codec,FieldsConsumer>();
- private final Set<String> fieldsSeen = new TreeSet<String>();
-
- public FieldsWriter(SegmentWriteState state) {
- this.state = state;
- }
-
- @Override
- public TermsConsumer addField(FieldInfo field) throws IOException {
- fieldsSeen.add(field.name);
- Codec codec = getCodec(field.name);
-
- FieldsConsumer fields = codecs.get(codec);
- if (fields == null) {
- fields = codec.fieldsConsumer(state);
- codecs.put(codec, fields);
- }
- return fields.addField(field);
- }
-
- @Override
- public void close() throws IOException {
- Iterator<FieldsConsumer> it = codecs.values().iterator();
- IOException err = null;
- while(it.hasNext()) {
- try {
- it.next().close();
- } catch (IOException ioe) {
- // keep first IOException we hit but keep
- // closing the rest
- if (err == null) {
- err = ioe;
- }
- }
- }
- if (err != null) {
- throw err;
- }
- }
- }
-
- private class FieldsReader extends FieldsProducer {
-
- private final Set<String> fields = new TreeSet<String>();
- private final Map<Codec,FieldsProducer> codecs = new HashMap<Codec,FieldsProducer>();
-
- public FieldsReader(Directory dir, FieldInfos fieldInfos,
- SegmentInfo si, int readBufferSize,
- int indexDivisor) throws IOException {
-
- final int fieldCount = fieldInfos.size();
- for(int i=0;i<fieldCount;i++) {
- FieldInfo fi = fieldInfos.fieldInfo(i);
- if (fi.isIndexed) {
- fields.add(fi.name);
- Codec codec = getCodec(fi.name);
- if (!codecs.containsKey(codec)) {
- codecs.put(codec, codec.fieldsProducer(new SegmentReadState(dir, si, fieldInfos, readBufferSize, indexDivisor)));
- }
- }
- }
- }
-
- private final class FieldsIterator extends FieldsEnum {
- private final Iterator<String> it;
- private String current;
-
- public FieldsIterator() {
- it = fields.iterator();
- }
-
- @Override
- public String next() {
- if (it.hasNext()) {
- current = it.next();
- } else {
- current = null;
- }
-
- return current;
- }
-
- @Override
- public TermsEnum terms() throws IOException {
- Terms terms = codecs.get(getCodec(current)).terms(current);
- if (terms != null) {
- return terms.iterator();
- } else {
- return null;
- }
- }
- }
-
- @Override
- public FieldsEnum iterator() throws IOException {
- return new FieldsIterator();
- }
-
- @Override
- public Terms terms(String field) throws IOException {
- Codec codec = getCodec(field);
-
- FieldsProducer fields = codecs.get(codec);
- assert fields != null;
- return fields.terms(field);
- }
-
- @Override
- public void close() throws IOException {
- Iterator<FieldsProducer> it = codecs.values().iterator();
- IOException err = null;
- while(it.hasNext()) {
- try {
- it.next().close();
- } catch (IOException ioe) {
- // keep first IOException we hit but keep
- // closing the rest
- if (err == null) {
- err = ioe;
- }
- }
- }
- if (err != null) {
- throw err;
- }
- }
-
- @Override
- public void loadTermsIndex(int indexDivisor) throws IOException {
- Iterator<FieldsProducer> it = codecs.values().iterator();
- while(it.hasNext()) {
- it.next().loadTermsIndex(indexDivisor);
- }
- }
- }
-
- public FieldsProducer fieldsProducer(SegmentReadState state)
- throws IOException {
- return new FieldsReader(state.dir, state.fieldInfos, state.segmentInfo, state.readBufferSize, state.termsIndexDivisor);
- }
-
- @Override
- public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
- Iterator<Codec> it = fields.values().iterator();
- Set<Codec> seen = new HashSet<Codec>();
- while(it.hasNext()) {
- final Codec codec = it.next();
- if (!seen.contains(codec)) {
- seen.add(codec);
- codec.files(dir, info, files);
- }
- }
- }
-
- @Override
- public void getExtensions(Set<String> extensions) {
- Iterator<Codec> it = fields.values().iterator();
- while(it.hasNext()) {
- final Codec codec = it.next();
- codec.getExtensions(extensions);
- }
- }
- }
-
public static class MyCodecs extends CodecProvider {
PerFieldCodecWrapper perField;