You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/08/15 19:01:34 UTC
svn commit: r1157910 - in /lucene/dev/branches/fieldtype:
lucene/src/java/org/apache/lucene/document/
lucene/src/test/org/apache/lucene/document/
modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
solr/src/java/org/apache/solr/handler...
Author: mikemccand
Date: Mon Aug 15 17:01:33 2011
New Revision: 1157910
URL: http://svn.apache.org/viewvc?rev=1157910&view=rev
Log:
LUCENE-2308: add missing classes
Added:
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/BinaryField.java (with props)
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldType.java (with props)
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/StringField.java (with props)
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/TextField.java (with props)
lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java (with props)
lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestDateTools.java (with props)
lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestDocument.java (with props)
Modified:
lucene/dev/branches/fieldtype/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java
lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/schema/FieldType.java
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/BinaryField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/BinaryField.java?rev=1157910&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/BinaryField.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/BinaryField.java Mon Aug 15 17:01:33 2011
@@ -0,0 +1,46 @@
+package org.apache.lucene.document;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public final class BinaryField extends Field {
+
+ public static final FieldType TYPE_STORED = new FieldType();
+ static {
+ TYPE_STORED.setStored(true);
+ TYPE_STORED.freeze();
+ }
+
+ public BinaryField(String name, byte[] value) {
+ super(name, BinaryField.TYPE_STORED, value);
+ this.isBinary = true;
+ }
+
+ public BinaryField(String name, byte[] value, int offset, int length) {
+ super(name, BinaryField.TYPE_STORED, value, offset, length);
+ this.isBinary = true;
+ }
+
+ public BinaryField(String name, FieldType custom, byte[] value) {
+ super(name, custom, value);
+ this.isBinary = true;
+ }
+
+ public boolean isNumeric() {
+ return false;
+ }
+}
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldType.java?rev=1157910&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldType.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldType.java Mon Aug 15 17:01:33 2011
@@ -0,0 +1,183 @@
+package org.apache.lucene.document;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class FieldType {
+
+ private boolean indexed;
+ private boolean stored;
+ private boolean tokenized;
+ private boolean storeTermVectors;
+ private boolean storeTermVectorOffsets;
+ private boolean storeTermVectorPositions;
+ private boolean omitNorms;
+ private boolean omitTermFreqsAndPositions;
+ private boolean lazy;
+ private boolean frozen;
+
+ public FieldType(FieldType ref) {
+ this.indexed = ref.indexed();
+ this.stored = ref.stored();
+ this.tokenized = ref.tokenized();
+ this.storeTermVectors = ref.storeTermVectors();
+ this.storeTermVectorOffsets = ref.storeTermVectorOffsets();
+ this.storeTermVectorPositions = ref.storeTermVectorPositions();
+ this.omitNorms = ref.omitNorms();
+ this.omitTermFreqsAndPositions = ref.omitTermFreqAndPositions();
+ this.lazy = ref.lazy();
+ }
+
+ public FieldType() {
+ }
+
+ private void checkIfFrozen() {
+ if (frozen) {
+ throw new IllegalStateException();
+ }
+ }
+
+ public void freeze() {
+ this.frozen = true;
+ }
+
+ public boolean indexed() {
+ return this.indexed;
+ }
+
+ public void setIndexed(boolean value) {
+ checkIfFrozen();
+ this.indexed = value;
+ }
+
+ public boolean stored() {
+ return this.stored;
+ }
+
+ public void setStored(boolean value) {
+ checkIfFrozen();
+ this.stored = value;
+ }
+
+ public boolean tokenized() {
+ return this.tokenized;
+ }
+
+ public void setTokenized(boolean value) {
+ checkIfFrozen();
+ this.tokenized = value;
+ }
+
+ public boolean storeTermVectors() {
+ return this.storeTermVectors;
+ }
+
+ public void setStoreTermVectors(boolean value) {
+ checkIfFrozen();
+ this.storeTermVectors = value;
+ }
+
+ public boolean storeTermVectorOffsets() {
+ return this.storeTermVectorOffsets;
+ }
+
+ public void setStoreTermVectorOffsets(boolean value) {
+ checkIfFrozen();
+ this.storeTermVectorOffsets = value;
+ }
+
+ public boolean storeTermVectorPositions() {
+ return this.storeTermVectorPositions;
+ }
+
+ public void setStoreTermVectorPositions(boolean value) {
+ checkIfFrozen();
+ this.storeTermVectorPositions = value;
+ }
+
+ public boolean omitNorms() {
+ return this.omitNorms;
+ }
+
+ public void setOmitNorms(boolean value) {
+ checkIfFrozen();
+ this.omitNorms = value;
+ }
+
+ public boolean omitTermFreqAndPositions() {
+ return this.omitTermFreqsAndPositions;
+ }
+
+ public void setOmitTermFreqAndPositions(boolean value) {
+ checkIfFrozen();
+ this.omitTermFreqsAndPositions = value;
+ }
+
+ public boolean lazy() {
+ return this.lazy;
+ }
+
+ public void setLazy(boolean value) {
+ checkIfFrozen();
+ this.lazy = value;
+ }
+
+ /** Prints a Field for human consumption. */
+ @Override
+ public final String toString() {
+ StringBuilder result = new StringBuilder();
+ if (stored()) {
+ result.append("stored");
+ }
+ if (indexed()) {
+ if (result.length() > 0)
+ result.append(",");
+ result.append("indexed");
+ }
+ if (tokenized()) {
+ if (result.length() > 0)
+ result.append(",");
+ result.append("tokenized");
+ }
+ if (storeTermVectors()) {
+ if (result.length() > 0)
+ result.append(",");
+ result.append("termVector");
+ }
+ if (storeTermVectorOffsets()) {
+ if (result.length() > 0)
+ result.append(",");
+ result.append("termVectorOffsets");
+ }
+ if (storeTermVectorPositions()) {
+ if (result.length() > 0)
+ result.append(",");
+ result.append("termVectorPosition");
+ }
+ if (omitNorms()) {
+ result.append(",omitNorms");
+ }
+ if (omitTermFreqAndPositions()) {
+ result.append(",omitTermFreqAndPositions");
+ }
+ if (lazy()){
+ result.append(",lazy");
+ }
+
+ return result.toString();
+ }
+}
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/StringField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/StringField.java?rev=1157910&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/StringField.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/StringField.java Mon Aug 15 17:01:33 2011
@@ -0,0 +1,53 @@
+package org.apache.lucene.document;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public final class StringField extends Field {
+
+ public static final FieldType TYPE_UNSTORED = new FieldType();
+ public static final FieldType TYPE_STORED = new FieldType();
+ static {
+ TYPE_UNSTORED.setIndexed(true);
+ TYPE_UNSTORED.setOmitNorms(true);
+ TYPE_UNSTORED.setOmitTermFreqAndPositions(true);
+ TYPE_UNSTORED.freeze();
+
+ TYPE_STORED.setIndexed(true);
+ TYPE_STORED.setStored(true);
+ TYPE_STORED.setOmitNorms(true);
+ TYPE_STORED.setOmitTermFreqAndPositions(true);
+ TYPE_STORED.freeze();
+ }
+
+ public StringField(String name, boolean internName, String value) {
+ super(name, StringField.TYPE_UNSTORED, value);
+ }
+
+ public StringField(String name, String value) {
+ this(name, true, value);
+ }
+
+ @Override
+ public String stringValue() {
+ return (fieldsData == null) ? null : fieldsData.toString();
+ }
+
+ public boolean isNumeric() {
+ return false;
+ }
+}
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/TextField.java?rev=1157910&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/TextField.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/TextField.java Mon Aug 15 17:01:33 2011
@@ -0,0 +1,54 @@
+package org.apache.lucene.document;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+
+import org.apache.lucene.analysis.TokenStream;
+
+public final class TextField extends Field {
+
+ public static final FieldType TYPE_UNSTORED = new FieldType();
+ public static final FieldType TYPE_STORED = new FieldType();
+ static {
+ TYPE_UNSTORED.setIndexed(true);
+ TYPE_UNSTORED.setTokenized(true);
+ TYPE_UNSTORED.freeze();
+
+ TYPE_STORED.setIndexed(true);
+ TYPE_STORED.setStored(true);
+ TYPE_STORED.setTokenized(true);
+ TYPE_STORED.freeze();
+ }
+
+ public TextField(String name, Reader reader) {
+ super(name, TextField.TYPE_UNSTORED, reader);
+ }
+
+ public TextField(String name, String value) {
+ super(name, TextField.TYPE_UNSTORED, value);
+ }
+
+ public TextField(String name, TokenStream stream) {
+ super(name, TextField.TYPE_UNSTORED, stream);
+ }
+
+ public boolean isNumeric() {
+ return false;
+ }
+}
Added: lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java?rev=1157910&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java Mon Aug 15 17:01:33 2011
@@ -0,0 +1,115 @@
+package org.apache.lucene.document;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+import org.apache.lucene.document.BinaryField;
+import org.apache.lucene.document.CompressionTools;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.store.Directory;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Tests {@link Document} class.
+ */
+public class TestBinaryDocument extends LuceneTestCase {
+
+ String binaryValStored = "this text will be stored as a byte array in the index";
+ String binaryValCompressed = "this text will be also stored and compressed as a byte array in the index";
+
+ public void testBinaryFieldInIndex()
+ throws Exception
+ {
+ FieldType ft = new FieldType();
+ ft.setStored(true);
+ IndexableField binaryFldStored = new BinaryField("binaryStored", binaryValStored.getBytes());
+ IndexableField stringFldStored = new Field("stringStored", ft, binaryValStored);
+
+ Document doc = new Document();
+
+ doc.add(binaryFldStored);
+
+ doc.add(stringFldStored);
+
+ /** test for field count */
+ assertEquals(2, doc.fields.size());
+
+ /** add the doc to a ram index */
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+ writer.addDocument(doc);
+
+ /** open a reader and fetch the document */
+ IndexReader reader = writer.getReader();
+ Document docFromReader = reader.document(0);
+ assertTrue(docFromReader != null);
+
+ /** fetch the binary stored field and compare it's content with the original one */
+ String binaryFldStoredTest = new String(docFromReader.getBinaryValue("binaryStored"));
+ assertTrue(binaryFldStoredTest.equals(binaryValStored));
+
+ /** fetch the string field and compare it's content with the original one */
+ String stringFldStoredTest = docFromReader.get("stringStored");
+ assertTrue(stringFldStoredTest.equals(binaryValStored));
+
+ writer.close();
+ reader.close();
+
+ reader = IndexReader.open(dir, false);
+ /** delete the document from index */
+ reader.deleteDocument(0);
+ assertEquals(0, reader.numDocs());
+
+ reader.close();
+ dir.close();
+ }
+
+ public void testCompressionTools() throws Exception {
+ IndexableField binaryFldCompressed = new BinaryField("binaryCompressed", CompressionTools.compress(binaryValCompressed.getBytes()));
+ IndexableField stringFldCompressed = new BinaryField("stringCompressed", CompressionTools.compressString(binaryValCompressed));
+
+ Document doc = new Document();
+
+ doc.add(binaryFldCompressed);
+ doc.add(stringFldCompressed);
+
+ /** add the doc to a ram index */
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+ writer.addDocument(doc);
+
+ /** open a reader and fetch the document */
+ IndexReader reader = writer.getReader();
+ Document docFromReader = reader.document(0);
+ assertTrue(docFromReader != null);
+
+ /** fetch the binary compressed field and compare it's content with the original one */
+ String binaryFldCompressedTest = new String(CompressionTools.decompress(docFromReader.getBinaryValue("binaryCompressed")));
+ assertTrue(binaryFldCompressedTest.equals(binaryValCompressed));
+ assertTrue(CompressionTools.decompressString(docFromReader.getBinaryValue("stringCompressed")).equals(binaryValCompressed));
+
+ writer.close();
+ reader.close();
+ dir.close();
+ }
+}
Added: lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestDateTools.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestDateTools.java?rev=1157910&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestDateTools.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestDateTools.java Mon Aug 15 17:01:33 2011
@@ -0,0 +1,200 @@
+package org.apache.lucene.document;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.TimeZone;
+import java.util.Locale;
+
+import org.apache.lucene.document.DateTools;
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class TestDateTools extends LuceneTestCase {
+
+ public void testStringToDate() throws ParseException {
+
+ Date d = null;
+ d = DateTools.stringToDate("2004");
+ assertEquals("2004-01-01 00:00:00:000", isoFormat(d));
+ d = DateTools.stringToDate("20040705");
+ assertEquals("2004-07-05 00:00:00:000", isoFormat(d));
+ d = DateTools.stringToDate("200407050910");
+ assertEquals("2004-07-05 09:10:00:000", isoFormat(d));
+ d = DateTools.stringToDate("20040705091055990");
+ assertEquals("2004-07-05 09:10:55:990", isoFormat(d));
+
+ try {
+ d = DateTools.stringToDate("97"); // no date
+ fail();
+ } catch(ParseException e) { /* expected exception */ }
+ try {
+ d = DateTools.stringToDate("200401011235009999"); // no date
+ fail();
+ } catch(ParseException e) { /* expected exception */ }
+ try {
+ d = DateTools.stringToDate("aaaa"); // no date
+ fail();
+ } catch(ParseException e) { /* expected exception */ }
+
+ }
+
+ public void testStringtoTime() throws ParseException {
+ long time = DateTools.stringToTime("197001010000");
+ Calendar cal = new GregorianCalendar();
+ cal.clear();
+ cal.set(1970, 0, 1, // year=1970, month=january, day=1
+ 0, 0, 0); // hour, minute, second
+ cal.set(Calendar.MILLISECOND, 0);
+ cal.setTimeZone(TimeZone.getTimeZone("GMT"));
+ assertEquals(cal.getTime().getTime(), time);
+ cal.set(1980, 1, 2, // year=1980, month=february, day=2
+ 11, 5, 0); // hour, minute, second
+ cal.set(Calendar.MILLISECOND, 0);
+ time = DateTools.stringToTime("198002021105");
+ assertEquals(cal.getTime().getTime(), time);
+ }
+
+ public void testDateAndTimetoString() throws ParseException {
+ Calendar cal = new GregorianCalendar();
+ cal.clear();
+ cal.setTimeZone(TimeZone.getTimeZone("GMT"));
+ cal.set(2004, 1, 3, // year=2004, month=february(!), day=3
+ 22, 8, 56); // hour, minute, second
+ cal.set(Calendar.MILLISECOND, 333);
+
+ String dateString;
+ dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.YEAR);
+ assertEquals("2004", dateString);
+ assertEquals("2004-01-01 00:00:00:000", isoFormat(DateTools.stringToDate(dateString)));
+
+ dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.MONTH);
+ assertEquals("200402", dateString);
+ assertEquals("2004-02-01 00:00:00:000", isoFormat(DateTools.stringToDate(dateString)));
+
+ dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.DAY);
+ assertEquals("20040203", dateString);
+ assertEquals("2004-02-03 00:00:00:000", isoFormat(DateTools.stringToDate(dateString)));
+
+ dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.HOUR);
+ assertEquals("2004020322", dateString);
+ assertEquals("2004-02-03 22:00:00:000", isoFormat(DateTools.stringToDate(dateString)));
+
+ dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.MINUTE);
+ assertEquals("200402032208", dateString);
+ assertEquals("2004-02-03 22:08:00:000", isoFormat(DateTools.stringToDate(dateString)));
+
+ dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.SECOND);
+ assertEquals("20040203220856", dateString);
+ assertEquals("2004-02-03 22:08:56:000", isoFormat(DateTools.stringToDate(dateString)));
+
+ dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.MILLISECOND);
+ assertEquals("20040203220856333", dateString);
+ assertEquals("2004-02-03 22:08:56:333", isoFormat(DateTools.stringToDate(dateString)));
+
+ // date before 1970:
+ cal.set(1961, 2, 5, // year=1961, month=march(!), day=5
+ 23, 9, 51); // hour, minute, second
+ cal.set(Calendar.MILLISECOND, 444);
+ dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.MILLISECOND);
+ assertEquals("19610305230951444", dateString);
+ assertEquals("1961-03-05 23:09:51:444", isoFormat(DateTools.stringToDate(dateString)));
+
+ dateString = DateTools.dateToString(cal.getTime(), DateTools.Resolution.HOUR);
+ assertEquals("1961030523", dateString);
+ assertEquals("1961-03-05 23:00:00:000", isoFormat(DateTools.stringToDate(dateString)));
+
+ // timeToString:
+ cal.set(1970, 0, 1, // year=1970, month=january, day=1
+ 0, 0, 0); // hour, minute, second
+ cal.set(Calendar.MILLISECOND, 0);
+ dateString = DateTools.timeToString(cal.getTime().getTime(),
+ DateTools.Resolution.MILLISECOND);
+ assertEquals("19700101000000000", dateString);
+
+ cal.set(1970, 0, 1, // year=1970, month=january, day=1
+ 1, 2, 3); // hour, minute, second
+ cal.set(Calendar.MILLISECOND, 0);
+ dateString = DateTools.timeToString(cal.getTime().getTime(),
+ DateTools.Resolution.MILLISECOND);
+ assertEquals("19700101010203000", dateString);
+ }
+
+ public void testRound() {
+ Calendar cal = new GregorianCalendar();
+ cal.clear();
+ cal.setTimeZone(TimeZone.getTimeZone("GMT"));
+ cal.set(2004, 1, 3, // year=2004, month=february(!), day=3
+ 22, 8, 56); // hour, minute, second
+ cal.set(Calendar.MILLISECOND, 333);
+ Date date = cal.getTime();
+ assertEquals("2004-02-03 22:08:56:333", isoFormat(date));
+
+ Date dateYear = DateTools.round(date, DateTools.Resolution.YEAR);
+ assertEquals("2004-01-01 00:00:00:000", isoFormat(dateYear));
+
+ Date dateMonth = DateTools.round(date, DateTools.Resolution.MONTH);
+ assertEquals("2004-02-01 00:00:00:000", isoFormat(dateMonth));
+
+ Date dateDay = DateTools.round(date, DateTools.Resolution.DAY);
+ assertEquals("2004-02-03 00:00:00:000", isoFormat(dateDay));
+
+ Date dateHour = DateTools.round(date, DateTools.Resolution.HOUR);
+ assertEquals("2004-02-03 22:00:00:000", isoFormat(dateHour));
+
+ Date dateMinute = DateTools.round(date, DateTools.Resolution.MINUTE);
+ assertEquals("2004-02-03 22:08:00:000", isoFormat(dateMinute));
+
+ Date dateSecond = DateTools.round(date, DateTools.Resolution.SECOND);
+ assertEquals("2004-02-03 22:08:56:000", isoFormat(dateSecond));
+
+ Date dateMillisecond = DateTools.round(date, DateTools.Resolution.MILLISECOND);
+ assertEquals("2004-02-03 22:08:56:333", isoFormat(dateMillisecond));
+
+ // long parameter:
+ long dateYearLong = DateTools.round(date.getTime(), DateTools.Resolution.YEAR);
+ assertEquals("2004-01-01 00:00:00:000", isoFormat(new Date(dateYearLong)));
+
+ long dateMillisecondLong = DateTools.round(date.getTime(), DateTools.Resolution.MILLISECOND);
+ assertEquals("2004-02-03 22:08:56:333", isoFormat(new Date(dateMillisecondLong)));
+ }
+
+ private String isoFormat(Date date) {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS", Locale.US);
+ sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+ return sdf.format(date);
+ }
+
+ public void testDateToolsUTC() throws Exception {
+ // Sun, 30 Oct 2005 00:00:00 +0000 -- the last second of 2005's DST in Europe/London
+ long time = 1130630400;
+ try {
+ TimeZone.setDefault(TimeZone.getTimeZone(/* "GMT" */ "Europe/London"));
+ String d1 = DateTools.dateToString(new Date(time*1000), DateTools.Resolution.MINUTE);
+ String d2 = DateTools.dateToString(new Date((time+3600)*1000), DateTools.Resolution.MINUTE);
+ assertFalse("different times", d1.equals(d2));
+ assertEquals("midnight", DateTools.stringToTime(d1), time*1000);
+ assertEquals("later", DateTools.stringToTime(d2), (time+3600)*1000);
+ } finally {
+ TimeZone.setDefault(null);
+ }
+ }
+
+}
Added: lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestDocument.java?rev=1157910&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestDocument.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/document/TestDocument.java Mon Aug 15 17:01:33 2011
@@ -0,0 +1,288 @@
+package org.apache.lucene.document;
+
+import org.apache.lucene.document.BinaryField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Tests {@link Document} class.
+ */
+public class TestDocument extends LuceneTestCase {
+
+ String binaryVal = "this text will be stored as a byte array in the index";
+ String binaryVal2 = "this text will be also stored as a byte array in the index";
+
+ public void testBinaryField() throws Exception {
+ Document doc = new Document();
+
+ FieldType ft = new FieldType();
+ ft.setStored(true);
+ IndexableField stringFld = new Field("string", ft, binaryVal);
+ IndexableField binaryFld = new BinaryField("binary", binaryVal.getBytes());
+ IndexableField binaryFld2 = new BinaryField("binary", binaryVal2.getBytes());
+
+ doc.add(stringFld);
+ doc.add(binaryFld);
+
+ assertEquals(2, doc.fields.size());
+
+ assertTrue(binaryFld.binaryValue(null) != null);
+ assertTrue(binaryFld.stored());
+ assertFalse(binaryFld.indexed());
+ assertFalse(binaryFld.tokenized());
+
+ String binaryTest = new String(doc.getBinaryValue("binary"));
+ assertTrue(binaryTest.equals(binaryVal));
+
+ String stringTest = doc.get("string");
+ assertTrue(binaryTest.equals(stringTest));
+
+ doc.add(binaryFld2);
+
+ assertEquals(3, doc.fields.size());
+
+ byte[][] binaryTests = doc.getBinaryValues("binary");
+
+ assertEquals(2, binaryTests.length);
+
+ binaryTest = new String(binaryTests[0]);
+ String binaryTest2 = new String(binaryTests[1]);
+
+ assertFalse(binaryTest.equals(binaryTest2));
+
+ assertTrue(binaryTest.equals(binaryVal));
+ assertTrue(binaryTest2.equals(binaryVal2));
+
+ doc.removeField("string");
+ assertEquals(2, doc.fields.size());
+
+ doc.removeFields("binary");
+ assertEquals(0, doc.fields.size());
+ }
+
+ /**
+ * Tests {@link Document#removeField(String)} method for a brand new Document
+ * that has not been indexed yet.
+ *
+ * @throws Exception on error
+ */
+ public void testRemoveForNewDocument() throws Exception {
+ Document doc = makeDocumentWithFields();
+ assertEquals(8, doc.fields.size());
+ doc.removeFields("keyword");
+ assertEquals(6, doc.fields.size());
+ doc.removeFields("doesnotexists"); // removing non-existing fields is
+ // siltenlty ignored
+ doc.removeFields("keyword"); // removing a field more than once
+ assertEquals(6, doc.fields.size());
+ doc.removeField("text");
+ assertEquals(5, doc.fields.size());
+ doc.removeField("text");
+ assertEquals(4, doc.fields.size());
+ doc.removeField("text");
+ assertEquals(4, doc.fields.size());
+ doc.removeField("doesnotexists"); // removing non-existing fields is
+ // siltenlty ignored
+ assertEquals(4, doc.fields.size());
+ doc.removeFields("unindexed");
+ assertEquals(2, doc.fields.size());
+ doc.removeFields("unstored");
+ assertEquals(0, doc.fields.size());
+ doc.removeFields("doesnotexists"); // removing non-existing fields is
+ // siltenlty ignored
+ assertEquals(0, doc.fields.size());
+ }
+
+ public void testConstructorExceptions() {
+ FieldType ft = new FieldType();
+ ft.setStored(true);
+ new Field("name", ft, "value"); // okay
+ new StringField("name", "value"); // okay
+ try {
+ new Field("name", new FieldType(), "value");
+ fail();
+ } catch (IllegalArgumentException e) {
+ // expected exception
+ }
+ new Field("name", ft, "value"); // okay
+ try {
+ FieldType ft2 = new FieldType();
+ ft2.setStored(true);
+ ft2.setStoreTermVectors(true);
+ new Field("name", ft2, "value");
+ fail();
+ } catch (IllegalArgumentException e) {
+ // expected exception
+ }
+ }
+
+ /**
+ * Tests {@link Document#getValues(String)} method for a brand new Document
+ * that has not been indexed yet.
+ *
+ * @throws Exception on error
+ */
+ public void testGetValuesForNewDocument() throws Exception {
+ doAssert(makeDocumentWithFields(), false);
+ }
+
+ /**
+ * Tests {@link Document#getValues(String)} method for a Document retrieved
+ * from an index.
+ *
+ * @throws Exception on error
+ */
+ public void testGetValuesForIndexedDocument() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+ writer.addDocument(makeDocumentWithFields());
+ IndexReader reader = writer.getReader();
+
+ IndexSearcher searcher = newSearcher(reader);
+
+ // search for something that does exists
+ Query query = new TermQuery(new Term("keyword", "test1"));
+
+ // ensure that queries return expected results without DateFilter first
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(1, hits.length);
+
+ doAssert(searcher.doc(hits[0].doc), true);
+ writer.close();
+ searcher.close();
+ reader.close();
+ dir.close();
+ }
+
+ private Document makeDocumentWithFields() {
+ Document doc = new Document();
+ FieldType stored = new FieldType();
+ stored.setStored(true);
+ doc.add(new Field("keyword", StringField.TYPE_STORED, "test1"));
+ doc.add(new Field("keyword", StringField.TYPE_STORED, "test2"));
+ doc.add(new Field("text", TextField.TYPE_STORED, "test1"));
+ doc.add(new Field("text", TextField.TYPE_STORED, "test2"));
+ doc.add(new Field("unindexed", stored, "test1"));
+ doc.add(new Field("unindexed", stored, "test2"));
+ doc
+ .add(new TextField("unstored", "test1"));
+ doc
+ .add(new TextField("unstored", "test2"));
+ return doc;
+ }
+
+ private void doAssert(Document doc, boolean fromIndex) {
+ IndexableField[] keywordFieldValues = doc.getFields("keyword");
+ IndexableField[] textFieldValues = doc.getFields("text");
+ IndexableField[] unindexedFieldValues = doc.getFields("unindexed");
+ IndexableField[] unstoredFieldValues = doc.getFields("unstored");
+
+ assertTrue(keywordFieldValues.length == 2);
+ assertTrue(textFieldValues.length == 2);
+ assertTrue(unindexedFieldValues.length == 2);
+ // this test cannot work for documents retrieved from the index
+ // since unstored fields will obviously not be returned
+ if (!fromIndex) {
+ assertTrue(unstoredFieldValues.length == 2);
+ }
+
+ assertTrue(keywordFieldValues[0].stringValue().equals("test1"));
+ assertTrue(keywordFieldValues[1].stringValue().equals("test2"));
+ assertTrue(textFieldValues[0].stringValue().equals("test1"));
+ assertTrue(textFieldValues[1].stringValue().equals("test2"));
+ assertTrue(unindexedFieldValues[0].stringValue().equals("test1"));
+ assertTrue(unindexedFieldValues[1].stringValue().equals("test2"));
+ // this test cannot work for documents retrieved from the index
+ // since unstored fields will obviously not be returned
+ if (!fromIndex) {
+ assertTrue(unstoredFieldValues[0].stringValue().equals("test1"));
+ assertTrue(unstoredFieldValues[1].stringValue().equals("test2"));
+ }
+ }
+
+ public void testFieldSetValue() throws Exception {
+
+ Field field = new Field("id", StringField.TYPE_STORED, "id1");
+ Document doc = new Document();
+ doc.add(field);
+ doc.add(new Field("keyword", StringField.TYPE_STORED, "test"));
+
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+ writer.addDocument(doc);
+ field.setValue("id2");
+ writer.addDocument(doc);
+ field.setValue("id3");
+ writer.addDocument(doc);
+
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+
+ Query query = new TermQuery(new Term("keyword", "test"));
+
+ // ensure that queries return expected results without DateFilter first
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(3, hits.length);
+ int result = 0;
+ for (int i = 0; i < 3; i++) {
+ Document doc2 = searcher.doc(hits[i].doc);
+ Field f = (Field) doc2.getField("id");
+ if (f.stringValue().equals("id1")) result |= 1;
+ else if (f.stringValue().equals("id2")) result |= 2;
+ else if (f.stringValue().equals("id3")) result |= 4;
+ else fail("unexpected id field");
+ }
+ writer.close();
+ searcher.close();
+ reader.close();
+ dir.close();
+ assertEquals("did not see all IDs", 7, result);
+ }
+
+ public void testFieldSetValueChangeBinary() {
+ Field field1 = new BinaryField("field1", new byte[0]);
+ Field field2 = new Field("field2", TextField.TYPE_STORED, "");
+ try {
+ field1.setValue("abc");
+ fail("did not hit expected exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ try {
+ field2.setValue(new byte[0]);
+ fail("did not hit expected exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ }
+}
Modified: lucene/dev/branches/fieldtype/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java?rev=1157910&r1=1157909&r2=1157910&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java (original)
+++ lucene/dev/branches/fieldtype/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java Mon Aug 15 17:01:33 2011
@@ -300,9 +300,9 @@ public abstract class ReadTask extends P
* @return A Collection of Field names (Strings)
*/
protected Collection<String> getFieldsToHighlight(Document document) {
- List<IndexableField> fieldables = document.getFields();
- Set<String> result = new HashSet<String>(fieldables.size());
- for (final IndexableField f : fieldables) {
+ List<IndexableField> fields = document.getFields();
+ Set<String> result = new HashSet<String>(fields.size());
+ for (final IndexableField f : fields) {
result.add(f.name());
}
return result;
Modified: lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1157910&r1=1157909&r2=1157910&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Mon Aug 15 17:01:33 2011
@@ -163,7 +163,7 @@ public class LukeRequestHandler extends
/**
- * @return a string representing a Fieldable's flags.
+ * @return a string representing a IndexableField's flags.
*/
private static String getFieldFlags( IndexableField f )
{
@@ -237,34 +237,34 @@ public class LukeRequestHandler extends
final CharsRef spare = new CharsRef();
SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<Object>();
for( Object o : doc.getFields() ) {
- Field fieldable = (Field)o;
+ Field field = (Field)o;
SimpleOrderedMap<Object> f = new SimpleOrderedMap<Object>();
- SchemaField sfield = schema.getFieldOrNull( fieldable.name() );
+ SchemaField sfield = schema.getFieldOrNull( field.name() );
FieldType ftype = (sfield==null)?null:sfield.getType();
f.add( "type", (ftype==null)?null:ftype.getTypeName() );
f.add( "schema", getFieldFlags( sfield ) );
- f.add( "flags", getFieldFlags( fieldable ) );
+ f.add( "flags", getFieldFlags( field ) );
- Term t = new Term(fieldable.name(), ftype!=null ? ftype.storedToIndexed(fieldable) : fieldable.stringValue());
+ Term t = new Term(field.name(), ftype!=null ? ftype.storedToIndexed(field) : field.stringValue());
- f.add( "value", (ftype==null)?null:ftype.toExternal( fieldable ) );
+ f.add( "value", (ftype==null)?null:ftype.toExternal( field ) );
// TODO: this really should be "stored"
- f.add( "internal", fieldable.stringValue() ); // may be a binary number
+ f.add( "internal", field.stringValue() ); // may be a binary number
- BytesRef bytes = fieldable.binaryValue(null);
+ BytesRef bytes = field.binaryValue(null);
if (bytes != null) {
f.add( "binary", Base64.byteArrayToBase64(bytes.bytes, bytes.offset, bytes.length));
}
- f.add( "boost", fieldable.boost() );
+ f.add( "boost", field.boost() );
f.add( "docFreq", t.text()==null ? 0 : reader.docFreq( t ) ); // this can be 0 for non-indexed fields
// If we have a term vector, return that
- if( fieldable.storeTermVectors() ) {
+ if( field.storeTermVectors() ) {
try {
- TermFreqVector v = reader.getTermFreqVector( docId, fieldable.name() );
+ TermFreqVector v = reader.getTermFreqVector( docId, field.name() );
if( v != null ) {
SimpleOrderedMap<Integer> tfv = new SimpleOrderedMap<Integer>();
for( int i=0; i<v.size(); i++ ) {
@@ -278,7 +278,7 @@ public class LukeRequestHandler extends
}
}
- finfo.add( fieldable.name(), f );
+ finfo.add( field.name(), f );
}
return finfo;
}
Modified: lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/schema/FieldType.java?rev=1157910&r1=1157909&r2=1157910&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/schema/FieldType.java Mon Aug 15 17:01:33 2011
@@ -262,40 +262,6 @@ public abstract class FieldType extends
return createField(field.getName(), val, newType, boost);
}
- /*protected Fieldable createField(String name, String val, org.apache.lucene.document.Field.Store storage, org.apache.lucene.document.Field.Index index,
- org.apache.lucene.document.Field.TermVector vec, boolean omitNorms, boolean omitTFPos, float boost){
- org.apache.lucene.document.Field f = new org.apache.lucene.document.Field(name,
- val,
- storage,
- index,
- vec);
- f.setOmitNorms(omitNorms);
- f.setOmitTermFreqAndPositions(omitTFPos);
- f.setBoost(boost);
- return f;
- }
-
- public Fieldable createField2(SchemaField field, Object value, float boost) {
- if (!field.indexed() && !field.stored()) {
- if (log.isTraceEnabled())
- log.trace("Ignoring unindexed/unstored field: " + field);
- return null;
- }
-
- String val;
- try {
- val = toInternal(value.toString());
- } catch (RuntimeException e) {
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error while creating field '" + field + "' from value '" + value + "'", e, false);
- }
- if (val==null) return null;
-
- return createField(field.getName(), val, getFieldStore(field, val),
- getFieldIndex(field, val), getFieldTermVec(field, val), field.omitNorms(),
- field.omitTf(), boost);
- }
- */
-
/**
* Create the field from native Lucene parts. Mostly intended for use by FieldTypes outputing multiple
* Fields per SchemaField