You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by bu...@apache.org on 2010/03/31 01:39:45 UTC
svn commit: r929340 - in /lucene/java/branches/flex_1458: ./
backwards/src/java/org/apache/lucene/store/
src/java/org/apache/lucene/index/ src/java/org/apache/lucene/store/
Author: buschmi
Date: Tue Mar 30 23:39:45 2010
New Revision: 929340
URL: http://svn.apache.org/viewvc?rev=929340&view=rev
Log:
LUCENE-2126: Split up IndexInput and IndexOutput and introduce DataInput and DataOutput.
Added:
lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataInput.java (with props)
lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataOutput.java (with props)
lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataInput.java (with props)
lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataOutput.java (with props)
Modified:
lucene/java/branches/flex_1458/CHANGES.txt
lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/IndexInput.java
lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/IndexOutput.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ByteSliceReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ByteSliceWriter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java
Modified: lucene/java/branches/flex_1458/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/CHANGES.txt?rev=929340&r1=929339&r2=929340&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/CHANGES.txt (original)
+++ lucene/java/branches/flex_1458/CHANGES.txt Tue Mar 30 23:39:45 2010
@@ -53,6 +53,10 @@ New features
representation for the in-memory terms dict index. (Mike
McCandless)
+* LUCENE-2126: Add new classes for data (de)serialization: DataInput
+ and DataOutput. IndexInput and IndexOutput extend these new classes.
+ (Michael Busch)
+
======================= Trunk (not yet released) =======================
Changes in backwards compatibility policy
Added: lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataInput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataInput.java?rev=929340&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataInput.java (added)
+++ lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataInput.java Tue Mar 30 23:39:45 2010
@@ -0,0 +1,234 @@
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Abstract base class for performing read operations of Lucene's low-level
+ * data types.
+ */
+public abstract class DataInput implements Cloneable {
+ private byte[] bytes; // used by readString()
+ private char[] chars; // used by readModifiedUTF8String()
+ private boolean preUTF8Strings; // true if we are reading old (modified UTF8) string format
+
+ /** Reads and returns a single byte.
+ * @see DataOutput#writeByte(byte)
+ */
+ public abstract byte readByte() throws IOException;
+
+ /** Reads a specified number of bytes into an array at the specified offset.
+ * @param b the array to read bytes into
+ * @param offset the offset in the array to start storing bytes
+ * @param len the number of bytes to read
+ * @see DataOutput#writeBytes(byte[],int)
+ */
+ public abstract void readBytes(byte[] b, int offset, int len)
+ throws IOException;
+
+ /** Reads a specified number of bytes into an array at the
+ * specified offset with control over whether the read
+ * should be buffered (callers who have their own buffer
+ * should pass in "false" for useBuffer). Currently only
+ * {@link BufferedIndexInput} respects this parameter.
+ * @param b the array to read bytes into
+ * @param offset the offset in the array to start storing bytes
+ * @param len the number of bytes to read
+ * @param useBuffer set to false if the caller will handle
+ * buffering.
+ * @see DataOutput#writeBytes(byte[],int)
+ */
+ public void readBytes(byte[] b, int offset, int len, boolean useBuffer)
+ throws IOException
+ {
+ // Default to ignoring useBuffer entirely
+ readBytes(b, offset, len);
+ }
+
+ /** Reads two bytes and returns a short.
+ * @see DataOutput#writeByte(byte)
+ */
+ public short readShort() throws IOException {
+ return (short) (((readByte() & 0xFF) << 8) | (readByte() & 0xFF));
+ }
+
+ /** Reads four bytes and returns an int.
+ * @see DataOutput#writeInt(int)
+ */
+ public int readInt() throws IOException {
+ return ((readByte() & 0xFF) << 24) | ((readByte() & 0xFF) << 16)
+ | ((readByte() & 0xFF) << 8) | (readByte() & 0xFF);
+ }
+
+ /** Reads an int stored in variable-length format. Reads between one and
+ * five bytes. Smaller values take fewer bytes. Negative numbers are not
+ * supported.
+ * @see DataOutput#writeVInt(int)
+ */
+ public int readVInt() throws IOException {
+ byte b = readByte();
+ int i = b & 0x7F;
+ for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+ b = readByte();
+ i |= (b & 0x7F) << shift;
+ }
+ return i;
+ }
+
+ /** Reads eight bytes and returns a long.
+ * @see DataOutput#writeLong(long)
+ */
+ public long readLong() throws IOException {
+ return (((long)readInt()) << 32) | (readInt() & 0xFFFFFFFFL);
+ }
+
+ /** Reads a long stored in variable-length format. Reads between one and
+ * nine bytes. Smaller values take fewer bytes. Negative numbers are not
+ * supported. */
+ public long readVLong() throws IOException {
+ byte b = readByte();
+ long i = b & 0x7F;
+ for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+ b = readByte();
+ i |= (b & 0x7FL) << shift;
+ }
+ return i;
+ }
+
+ /** Call this if readString should read characters stored
+ * in the old modified UTF8 format (length in java chars
+ * and java's modified UTF8 encoding). This is used for
+ * indices written pre-2.4 See LUCENE-510 for details. */
+ public void setModifiedUTF8StringsMode() {
+ preUTF8Strings = true;
+ }
+
+ /** Reads a string.
+ * @see DataOutput#writeString(String)
+ */
+ public String readString() throws IOException {
+ if (preUTF8Strings)
+ return readModifiedUTF8String();
+ int length = readVInt();
+ if (bytes == null || length > bytes.length)
+ bytes = new byte[(int) (length*1.25)];
+ readBytes(bytes, 0, length);
+ return new String(bytes, 0, length, "UTF-8");
+ }
+
+ private String readModifiedUTF8String() throws IOException {
+ int length = readVInt();
+ if (chars == null || length > chars.length)
+ chars = new char[length];
+ readChars(chars, 0, length);
+ return new String(chars, 0, length);
+ }
+
+ /** Reads Lucene's old "modified UTF-8" encoded
+ * characters into an array.
+ * @param buffer the array to read characters into
+ * @param start the offset in the array to start storing characters
+ * @param length the number of characters to read
+ * @see DataOutput#writeChars(String,int,int)
+ * @deprecated -- please use readString or readBytes
+ * instead, and construct the string
+ * from those utf8 bytes
+ */
+ @Deprecated
+ public void readChars(char[] buffer, int start, int length)
+ throws IOException {
+ final int end = start + length;
+ for (int i = start; i < end; i++) {
+ byte b = readByte();
+ if ((b & 0x80) == 0)
+ buffer[i] = (char)(b & 0x7F);
+ else if ((b & 0xE0) != 0xE0) {
+ buffer[i] = (char)(((b & 0x1F) << 6)
+ | (readByte() & 0x3F));
+ } else {
+ buffer[i] = (char)(((b & 0x0F) << 12)
+ | ((readByte() & 0x3F) << 6)
+ | (readByte() & 0x3F));
+ }
+ }
+ }
+
+ /**
+ * Expert
+ *
+ * Similar to {@link #readChars(char[], int, int)} but does not do any conversion operations on the bytes it is reading in. It still
+ * has to invoke {@link #readByte()} just as {@link #readChars(char[], int, int)} does, but it does not need a buffer to store anything
+ * and it does not have to do any of the bitwise operations, since we don't actually care what is in the byte except to determine
+ * how many more bytes to read
+ * @param length The number of chars to read
+ * @deprecated this method operates on old "modified utf8" encoded
+ * strings
+ */
+ @Deprecated
+ public void skipChars(int length) throws IOException{
+ for (int i = 0; i < length; i++) {
+ byte b = readByte();
+ if ((b & 0x80) == 0){
+ //do nothing, we only need one byte
+ } else if ((b & 0xE0) != 0xE0) {
+ readByte();//read an additional byte
+ } else {
+ //read two additional bytes.
+ readByte();
+ readByte();
+ }
+ }
+ }
+
+ /** Returns a clone of this stream.
+ *
+ * <p>Clones of a stream access the same data, and are positioned at the same
+ * point as the stream they were cloned from.
+ *
+ * <p>Expert: Subclasses must ensure that clones may be positioned at
+ * different points in the input from each other and from the stream they
+ * were cloned from.
+ */
+ @Override
+ public Object clone() {
+ DataInput clone = null;
+ try {
+ clone = (DataInput)super.clone();
+ } catch (CloneNotSupportedException e) {}
+
+ clone.bytes = null;
+ clone.chars = null;
+
+ return clone;
+ }
+
+ public Map<String,String> readStringStringMap() throws IOException {
+ final Map<String,String> map = new HashMap<String,String>();
+ final int count = readInt();
+ for(int i=0;i<count;i++) {
+ final String key = readString();
+ final String val = readString();
+ map.put(key, val);
+ }
+
+ return map;
+ }
+}
Propchange: lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataInput.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataOutput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataOutput.java?rev=929340&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataOutput.java (added)
+++ lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataOutput.java Tue Mar 30 23:39:45 2010
@@ -0,0 +1,194 @@
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.UnicodeUtil;
+
+/**
+ * Abstract base class for performing write operations of Lucene's low-level
+ * data types.
+ */
+public abstract class DataOutput {
+
+ private BytesRef utf8Result = new BytesRef(10);
+
+ /** Writes a single byte.
+ * @see IndexInput#readByte()
+ */
+ public abstract void writeByte(byte b) throws IOException;
+
+ /** Writes an array of bytes.
+ * @param b the bytes to write
+ * @param length the number of bytes to write
+ * @see DataInput#readBytes(byte[],int,int)
+ */
+ public void writeBytes(byte[] b, int length) throws IOException {
+ writeBytes(b, 0, length);
+ }
+
+ /** Writes an array of bytes.
+ * @param b the bytes to write
+ * @param offset the offset in the byte array
+ * @param length the number of bytes to write
+ * @see DataInput#readBytes(byte[],int,int)
+ */
+ public abstract void writeBytes(byte[] b, int offset, int length) throws IOException;
+
+ /** Writes an int as four bytes.
+ * @see DataInput#readInt()
+ */
+ public void writeInt(int i) throws IOException {
+ writeByte((byte)(i >> 24));
+ writeByte((byte)(i >> 16));
+ writeByte((byte)(i >> 8));
+ writeByte((byte) i);
+ }
+
+ /** Writes an int in a variable-length format. Writes between one and
+ * five bytes. Smaller values take fewer bytes. Negative numbers are not
+ * supported.
+ * @see DataInput#readVInt()
+ */
+ public void writeVInt(int i) throws IOException {
+ while ((i & ~0x7F) != 0) {
+ writeByte((byte)((i & 0x7f) | 0x80));
+ i >>>= 7;
+ }
+ writeByte((byte)i);
+ }
+
+ /** Writes a long as eight bytes.
+ * @see DataInput#readLong()
+ */
+ public void writeLong(long i) throws IOException {
+ writeInt((int) (i >> 32));
+ writeInt((int) i);
+ }
+
+ /** Writes an long in a variable-length format. Writes between one and five
+ * bytes. Smaller values take fewer bytes. Negative numbers are not
+ * supported.
+ * @see DataInput#readVLong()
+ */
+ public void writeVLong(long i) throws IOException {
+ while ((i & ~0x7F) != 0) {
+ writeByte((byte)((i & 0x7f) | 0x80));
+ i >>>= 7;
+ }
+ writeByte((byte)i);
+ }
+
+ /** Writes a string.
+ * @see DataInput#readString()
+ */
+ public void writeString(String s) throws IOException {
+ UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
+ writeVInt(utf8Result.length);
+ writeBytes(utf8Result.bytes, 0, utf8Result.length);
+ }
+
+ /** Writes a sub sequence of characters from s as the old
+ * format (modified UTF-8 encoded bytes).
+ * @param s the source of the characters
+ * @param start the first character in the sequence
+ * @param length the number of characters in the sequence
+ * @deprecated -- please pre-convert to utf8 bytes
+ * instead or use {@link #writeString}
+ */
+ @Deprecated
+ public void writeChars(String s, int start, int length)
+ throws IOException {
+ final int end = start + length;
+ for (int i = start; i < end; i++) {
+ final int code = s.charAt(i);
+ if (code >= 0x01 && code <= 0x7F)
+ writeByte((byte)code);
+ else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
+ writeByte((byte)(0xC0 | (code >> 6)));
+ writeByte((byte)(0x80 | (code & 0x3F)));
+ } else {
+ writeByte((byte)(0xE0 | (code >>> 12)));
+ writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
+ writeByte((byte)(0x80 | (code & 0x3F)));
+ }
+ }
+ }
+
+ /** Writes a sub sequence of characters from char[] as
+ * the old format (modified UTF-8 encoded bytes).
+ * @param s the source of the characters
+ * @param start the first character in the sequence
+ * @param length the number of characters in the sequence
+ * @deprecated -- please pre-convert to utf8 bytes instead or use {@link #writeString}
+ */
+ @Deprecated
+ public void writeChars(char[] s, int start, int length)
+ throws IOException {
+ final int end = start + length;
+ for (int i = start; i < end; i++) {
+ final int code = s[i];
+ if (code >= 0x01 && code <= 0x7F)
+ writeByte((byte)code);
+ else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
+ writeByte((byte)(0xC0 | (code >> 6)));
+ writeByte((byte)(0x80 | (code & 0x3F)));
+ } else {
+ writeByte((byte)(0xE0 | (code >>> 12)));
+ writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
+ writeByte((byte)(0x80 | (code & 0x3F)));
+ }
+ }
+ }
+
+ private static int COPY_BUFFER_SIZE = 16384;
+ private byte[] copyBuffer;
+
+ /** Copy numBytes bytes from input to ourself. */
+ public void copyBytes(DataInput input, long numBytes) throws IOException {
+ assert numBytes >= 0: "numBytes=" + numBytes;
+ long left = numBytes;
+ if (copyBuffer == null)
+ copyBuffer = new byte[COPY_BUFFER_SIZE];
+ while(left > 0) {
+ final int toCopy;
+ if (left > COPY_BUFFER_SIZE)
+ toCopy = COPY_BUFFER_SIZE;
+ else
+ toCopy = (int) left;
+ input.readBytes(copyBuffer, 0, toCopy);
+ writeBytes(copyBuffer, 0, toCopy);
+ left -= toCopy;
+ }
+ }
+
+ public void writeStringStringMap(Map<String,String> map) throws IOException {
+ if (map == null) {
+ writeInt(0);
+ } else {
+ writeInt(map.size());
+ for(final Map.Entry<String, String> entry: map.entrySet()) {
+ writeString(entry.getKey());
+ writeString(entry.getValue());
+ }
+ }
+ }
+}
Propchange: lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/DataOutput.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/IndexInput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/IndexInput.java?rev=929340&r1=929339&r2=929340&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/IndexInput.java (original)
+++ lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/IndexInput.java Tue Mar 30 23:39:45 2010
@@ -17,180 +17,14 @@ package org.apache.lucene.store;
* limitations under the License.
*/
-import java.io.IOException;
import java.io.Closeable;
-import java.util.Map;
-import java.util.HashMap;
+import java.io.IOException;
/** Abstract base class for input from a file in a {@link Directory}. A
* random-access input stream. Used for all Lucene index input operations.
* @see Directory
*/
-public abstract class IndexInput implements Cloneable,Closeable {
- private byte[] bytes; // used by readString()
- private char[] chars; // used by readModifiedUTF8String()
- private boolean preUTF8Strings; // true if we are reading old (modified UTF8) string format
-
- /** Reads and returns a single byte.
- * @see IndexOutput#writeByte(byte)
- */
- public abstract byte readByte() throws IOException;
-
- /** Reads a specified number of bytes into an array at the specified offset.
- * @param b the array to read bytes into
- * @param offset the offset in the array to start storing bytes
- * @param len the number of bytes to read
- * @see IndexOutput#writeBytes(byte[],int)
- */
- public abstract void readBytes(byte[] b, int offset, int len)
- throws IOException;
-
- /** Reads a specified number of bytes into an array at the
- * specified offset with control over whether the read
- * should be buffered (callers who have their own buffer
- * should pass in "false" for useBuffer). Currently only
- * {@link BufferedIndexInput} respects this parameter.
- * @param b the array to read bytes into
- * @param offset the offset in the array to start storing bytes
- * @param len the number of bytes to read
- * @param useBuffer set to false if the caller will handle
- * buffering.
- * @see IndexOutput#writeBytes(byte[],int)
- */
- public void readBytes(byte[] b, int offset, int len, boolean useBuffer)
- throws IOException
- {
- // Default to ignoring useBuffer entirely
- readBytes(b, offset, len);
- }
-
- /** Reads four bytes and returns an int.
- * @see IndexOutput#writeInt(int)
- */
- public int readInt() throws IOException {
- return ((readByte() & 0xFF) << 24) | ((readByte() & 0xFF) << 16)
- | ((readByte() & 0xFF) << 8) | (readByte() & 0xFF);
- }
-
- /** Reads an int stored in variable-length format. Reads between one and
- * five bytes. Smaller values take fewer bytes. Negative numbers are not
- * supported.
- * @see IndexOutput#writeVInt(int)
- */
- public int readVInt() throws IOException {
- byte b = readByte();
- int i = b & 0x7F;
- for (int shift = 7; (b & 0x80) != 0; shift += 7) {
- b = readByte();
- i |= (b & 0x7F) << shift;
- }
- return i;
- }
-
- /** Reads eight bytes and returns a long.
- * @see IndexOutput#writeLong(long)
- */
- public long readLong() throws IOException {
- return (((long)readInt()) << 32) | (readInt() & 0xFFFFFFFFL);
- }
-
- /** Reads a long stored in variable-length format. Reads between one and
- * nine bytes. Smaller values take fewer bytes. Negative numbers are not
- * supported. */
- public long readVLong() throws IOException {
- byte b = readByte();
- long i = b & 0x7F;
- for (int shift = 7; (b & 0x80) != 0; shift += 7) {
- b = readByte();
- i |= (b & 0x7FL) << shift;
- }
- return i;
- }
-
- /** Call this if readString should read characters stored
- * in the old modified UTF8 format (length in java chars
- * and java's modified UTF8 encoding). This is used for
- * indices written pre-2.4 See LUCENE-510 for details. */
- public void setModifiedUTF8StringsMode() {
- preUTF8Strings = true;
- }
-
- /** Reads a string.
- * @see IndexOutput#writeString(String)
- */
- public String readString() throws IOException {
- if (preUTF8Strings)
- return readModifiedUTF8String();
- int length = readVInt();
- if (bytes == null || length > bytes.length)
- bytes = new byte[(int) (length*1.25)];
- readBytes(bytes, 0, length);
- return new String(bytes, 0, length, "UTF-8");
- }
-
- private String readModifiedUTF8String() throws IOException {
- int length = readVInt();
- if (chars == null || length > chars.length)
- chars = new char[length];
- readChars(chars, 0, length);
- return new String(chars, 0, length);
- }
-
- /** Reads Lucene's old "modified UTF-8" encoded
- * characters into an array.
- * @param buffer the array to read characters into
- * @param start the offset in the array to start storing characters
- * @param length the number of characters to read
- * @see IndexOutput#writeChars(String,int,int)
- * @deprecated -- please use readString or readBytes
- * instead, and construct the string
- * from those utf8 bytes
- */
- public void readChars(char[] buffer, int start, int length)
- throws IOException {
- final int end = start + length;
- for (int i = start; i < end; i++) {
- byte b = readByte();
- if ((b & 0x80) == 0)
- buffer[i] = (char)(b & 0x7F);
- else if ((b & 0xE0) != 0xE0) {
- buffer[i] = (char)(((b & 0x1F) << 6)
- | (readByte() & 0x3F));
- } else
- buffer[i] = (char)(((b & 0x0F) << 12)
- | ((readByte() & 0x3F) << 6)
- | (readByte() & 0x3F));
- }
- }
-
- /**
- * Expert
- *
- * Similar to {@link #readChars(char[], int, int)} but does not do any conversion operations on the bytes it is reading in. It still
- * has to invoke {@link #readByte()} just as {@link #readChars(char[], int, int)} does, but it does not need a buffer to store anything
- * and it does not have to do any of the bitwise operations, since we don't actually care what is in the byte except to determine
- * how many more bytes to read
- * @param length The number of chars to read
- * @deprecated this method operates on old "modified utf8" encoded
- * strings
- */
- public void skipChars(int length) throws IOException{
- for (int i = 0; i < length; i++) {
- byte b = readByte();
- if ((b & 0x80) == 0){
- //do nothing, we only need one byte
- }
- else if ((b & 0xE0) != 0xE0) {
- readByte();//read an additional byte
- } else{
- //read two additional bytes.
- readByte();
- readByte();
- }
- }
- }
-
-
+public abstract class IndexInput extends DataInput implements Cloneable,Closeable {
/** Closes the stream to further operations. */
public abstract void close() throws IOException;
@@ -207,38 +41,4 @@ public abstract class IndexInput impleme
/** The number of bytes in the file. */
public abstract long length();
-
- /** Returns a clone of this stream.
- *
- * <p>Clones of a stream access the same data, and are positioned at the same
- * point as the stream they were cloned from.
- *
- * <p>Expert: Subclasses must ensure that clones may be positioned at
- * different points in the input from each other and from the stream they
- * were cloned from.
- */
- @Override
- public Object clone() {
- IndexInput clone = null;
- try {
- clone = (IndexInput)super.clone();
- } catch (CloneNotSupportedException e) {}
-
- clone.bytes = null;
- clone.chars = null;
-
- return clone;
- }
-
- public Map<String,String> readStringStringMap() throws IOException {
- final Map<String,String> map = new HashMap<String,String>();
- final int count = readInt();
- for(int i=0;i<count;i++) {
- final String key = readString();
- final String val = readString();
- map.put(key, val);
- }
-
- return map;
- }
-}
+}
\ No newline at end of file
Modified: lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/IndexOutput.java?rev=929340&r1=929339&r2=929340&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/java/branches/flex_1458/backwards/src/java/org/apache/lucene/store/IndexOutput.java Tue Mar 30 23:39:45 2010
@@ -17,166 +17,15 @@ package org.apache.lucene.store;
* limitations under the License.
*/
-import java.io.IOException;
import java.io.Closeable;
-import java.util.Map;
-import org.apache.lucene.util.UnicodeUtil;
+import java.io.IOException;
/** Abstract base class for output to a file in a Directory. A random-access
* output stream. Used for all Lucene index output operations.
* @see Directory
* @see IndexInput
*/
-public abstract class IndexOutput implements Closeable {
-
- private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result();
-
- /** Writes a single byte.
- * @see IndexInput#readByte()
- */
- public abstract void writeByte(byte b) throws IOException;
-
- /** Writes an array of bytes.
- * @param b the bytes to write
- * @param length the number of bytes to write
- * @see IndexInput#readBytes(byte[],int,int)
- */
- public void writeBytes(byte[] b, int length) throws IOException {
- writeBytes(b, 0, length);
- }
-
- /** Writes an array of bytes.
- * @param b the bytes to write
- * @param offset the offset in the byte array
- * @param length the number of bytes to write
- * @see IndexInput#readBytes(byte[],int,int)
- */
- public abstract void writeBytes(byte[] b, int offset, int length) throws IOException;
-
- /** Writes an int as four bytes.
- * @see IndexInput#readInt()
- */
- public void writeInt(int i) throws IOException {
- writeByte((byte)(i >> 24));
- writeByte((byte)(i >> 16));
- writeByte((byte)(i >> 8));
- writeByte((byte) i);
- }
-
- /** Writes an int in a variable-length format. Writes between one and
- * five bytes. Smaller values take fewer bytes. Negative numbers are not
- * supported.
- * @see IndexInput#readVInt()
- */
- public void writeVInt(int i) throws IOException {
- while ((i & ~0x7F) != 0) {
- writeByte((byte)((i & 0x7f) | 0x80));
- i >>>= 7;
- }
- writeByte((byte)i);
- }
-
- /** Writes a long as eight bytes.
- * @see IndexInput#readLong()
- */
- public void writeLong(long i) throws IOException {
- writeInt((int) (i >> 32));
- writeInt((int) i);
- }
-
- /** Writes an long in a variable-length format. Writes between one and five
- * bytes. Smaller values take fewer bytes. Negative numbers are not
- * supported.
- * @see IndexInput#readVLong()
- */
- public void writeVLong(long i) throws IOException {
- while ((i & ~0x7F) != 0) {
- writeByte((byte)((i & 0x7f) | 0x80));
- i >>>= 7;
- }
- writeByte((byte)i);
- }
-
- /** Writes a string.
- * @see IndexInput#readString()
- */
- public void writeString(String s) throws IOException {
- UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
- writeVInt(utf8Result.length);
- writeBytes(utf8Result.result, 0, utf8Result.length);
- }
-
- /** Writes a sub sequence of characters from s as the old
- * format (modified UTF-8 encoded bytes).
- * @param s the source of the characters
- * @param start the first character in the sequence
- * @param length the number of characters in the sequence
- * @deprecated -- please pre-convert to utf8 bytes
- * instead or use {@link #writeString}
- */
- public void writeChars(String s, int start, int length)
- throws IOException {
- final int end = start + length;
- for (int i = start; i < end; i++) {
- final int code = (int)s.charAt(i);
- if (code >= 0x01 && code <= 0x7F)
- writeByte((byte)code);
- else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
- writeByte((byte)(0xC0 | (code >> 6)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- } else {
- writeByte((byte)(0xE0 | (code >>> 12)));
- writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- }
- }
- }
-
- /** Writes a sub sequence of characters from char[] as
- * the old format (modified UTF-8 encoded bytes).
- * @param s the source of the characters
- * @param start the first character in the sequence
- * @param length the number of characters in the sequence
- * @deprecated -- please pre-convert to utf8 bytes instead or use {@link #writeString}
- */
- public void writeChars(char[] s, int start, int length)
- throws IOException {
- final int end = start + length;
- for (int i = start; i < end; i++) {
- final int code = (int)s[i];
- if (code >= 0x01 && code <= 0x7F)
- writeByte((byte)code);
- else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
- writeByte((byte)(0xC0 | (code >> 6)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- } else {
- writeByte((byte)(0xE0 | (code >>> 12)));
- writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- }
- }
- }
-
- private static int COPY_BUFFER_SIZE = 16384;
- private byte[] copyBuffer;
-
- /** Copy numBytes bytes from input to ourself. */
- public void copyBytes(IndexInput input, long numBytes) throws IOException {
- assert numBytes >= 0: "numBytes=" + numBytes;
- long left = numBytes;
- if (copyBuffer == null)
- copyBuffer = new byte[COPY_BUFFER_SIZE];
- while(left > 0) {
- final int toCopy;
- if (left > COPY_BUFFER_SIZE)
- toCopy = COPY_BUFFER_SIZE;
- else
- toCopy = (int) left;
- input.readBytes(copyBuffer, 0, toCopy);
- writeBytes(copyBuffer, 0, toCopy);
- left -= toCopy;
- }
- }
+public abstract class IndexOutput extends DataOutput implements Closeable {
/** Forces any buffered output to be written. */
public abstract void flush() throws IOException;
@@ -208,17 +57,5 @@ public abstract class IndexOutput implem
* undefined. Otherwise the file is truncated.
* @param length file length
*/
- public void setLength(long length) throws IOException {};
-
- public void writeStringStringMap(Map<String,String> map) throws IOException {
- if (map == null) {
- writeInt(0);
- } else {
- writeInt(map.size());
- for(final Map.Entry<String, String> entry: map.entrySet()) {
- writeString(entry.getKey());
- writeString(entry.getValue());
- }
- }
- }
+ public void setLength(long length) throws IOException {}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ByteSliceReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ByteSliceReader.java?rev=929340&r1=929339&r2=929340&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ByteSliceReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ByteSliceReader.java Tue Mar 30 23:39:45 2010
@@ -17,16 +17,17 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
import java.io.IOException;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
/* IndexInput that knows how to read the byte slices written
* by Posting and PostingVector. We read the bytes in
* each slice until we hit the end of that slice at which
* point we read the forwarding address of the next slice
* and then jump to it.*/
-final class ByteSliceReader extends IndexInput {
+final class ByteSliceReader extends DataInput {
ByteBlockPool pool;
int bufferUpto;
byte[] buffer;
@@ -75,7 +76,7 @@ final class ByteSliceReader extends Inde
return buffer[upto++];
}
- public long writeTo(IndexOutput out) throws IOException {
+ public long writeTo(DataOutput out) throws IOException {
long size = 0;
while(true) {
if (limit + bufferOffset == endIndex) {
@@ -136,14 +137,4 @@ final class ByteSliceReader extends Inde
}
}
}
-
- @Override
- public long getFilePointer() {throw new RuntimeException("not implemented");}
- @Override
- public long length() {throw new RuntimeException("not implemented");}
- @Override
- public void seek(long pos) {throw new RuntimeException("not implemented");}
- @Override
- public void close() {throw new RuntimeException("not implemented");}
-}
-
+}
\ No newline at end of file
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ByteSliceWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ByteSliceWriter.java?rev=929340&r1=929339&r2=929340&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ByteSliceWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ByteSliceWriter.java Tue Mar 30 23:39:45 2010
@@ -1,5 +1,7 @@
package org.apache.lucene.index;
+import org.apache.lucene.store.DataOutput;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -24,7 +26,7 @@ package org.apache.lucene.index;
* posting list for many terms in RAM.
*/
-final class ByteSliceWriter {
+final class ByteSliceWriter extends DataOutput {
private byte[] slice;
private int upto;
@@ -38,7 +40,7 @@ final class ByteSliceWriter {
/**
* Set up the writer to write at address.
- */
+ */
public void init(int address) {
slice = pool.buffers[address >> DocumentsWriter.BYTE_BLOCK_SHIFT];
assert slice != null;
@@ -48,6 +50,7 @@ final class ByteSliceWriter {
}
/** Write byte into byte slice stream */
+ @Override
public void writeByte(byte b) {
assert slice != null;
if (slice[upto] != 0) {
@@ -60,6 +63,7 @@ final class ByteSliceWriter {
assert upto != slice.length;
}
+ @Override
public void writeBytes(final byte[] b, int offset, final int len) {
final int offsetEnd = offset + len;
while(offset < offsetEnd) {
@@ -78,12 +82,4 @@ final class ByteSliceWriter {
public int getAddress() {
return upto + (offset0 & DocumentsWriter.BYTE_BLOCK_NOT_MASK);
}
-
- public void writeVInt(int i) {
- while ((i & ~0x7F) != 0) {
- writeByte((byte)((i & 0x7f) | 0x80));
- i >>>= 7;
- }
- writeByte((byte) i);
- }
-}
+}
\ No newline at end of file
Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataInput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataInput.java?rev=929340&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataInput.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataInput.java Tue Mar 30 23:39:45 2010
@@ -0,0 +1,239 @@
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
+
+/**
+ * Abstract base class for performing read operations of Lucene's low-level
+ * data types.
+ */
+public abstract class DataInput implements Cloneable {
+ private byte[] bytes; // used by readString()
+ private char[] chars; // used by readModifiedUTF8String()
+ private boolean preUTF8Strings; // true if we are reading old (modified UTF8) string format
+
+ /** Reads and returns a single byte.
+ * @see DataOutput#writeByte(byte)
+ */
+ public abstract byte readByte() throws IOException;
+
+ /** Reads a specified number of bytes into an array at the specified offset.
+ * @param b the array to read bytes into
+ * @param offset the offset in the array to start storing bytes
+ * @param len the number of bytes to read
+ * @see DataOutput#writeBytes(byte[],int)
+ */
+ public abstract void readBytes(byte[] b, int offset, int len)
+ throws IOException;
+
+ /** Reads a specified number of bytes into an array at the
+ * specified offset with control over whether the read
+ * should be buffered (callers who have their own buffer
+ * should pass in "false" for useBuffer). Currently only
+ * {@link BufferedIndexInput} respects this parameter.
+ * @param b the array to read bytes into
+ * @param offset the offset in the array to start storing bytes
+ * @param len the number of bytes to read
+ * @param useBuffer set to false if the caller will handle
+ * buffering.
+ * @see DataOutput#writeBytes(byte[],int)
+ */
+ public void readBytes(byte[] b, int offset, int len, boolean useBuffer)
+ throws IOException
+ {
+ // Default to ignoring useBuffer entirely
+ readBytes(b, offset, len);
+ }
+
+ /** Reads two bytes and returns a short.
+ * @see DataOutput#writeByte(byte)
+ */
+ public short readShort() throws IOException {
+ return (short) (((readByte() & 0xFF) << 8) | (readByte() & 0xFF));
+ }
+
+ /** Reads four bytes and returns an int.
+ * @see DataOutput#writeInt(int)
+ */
+ public int readInt() throws IOException {
+ return ((readByte() & 0xFF) << 24) | ((readByte() & 0xFF) << 16)
+ | ((readByte() & 0xFF) << 8) | (readByte() & 0xFF);
+ }
+
+ /** Reads an int stored in variable-length format. Reads between one and
+ * five bytes. Smaller values take fewer bytes. Negative numbers are not
+ * supported.
+ * @see DataOutput#writeVInt(int)
+ */
+ public int readVInt() throws IOException {
+ byte b = readByte();
+ int i = b & 0x7F;
+ for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+ b = readByte();
+ i |= (b & 0x7F) << shift;
+ }
+ return i;
+ }
+
+ /** Reads eight bytes and returns a long.
+ * @see DataOutput#writeLong(long)
+ */
+ public long readLong() throws IOException {
+ return (((long)readInt()) << 32) | (readInt() & 0xFFFFFFFFL);
+ }
+
+ /** Reads a long stored in variable-length format. Reads between one and
+ * nine bytes. Smaller values take fewer bytes. Negative numbers are not
+ * supported. */
+ public long readVLong() throws IOException {
+ byte b = readByte();
+ long i = b & 0x7F;
+ for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+ b = readByte();
+ i |= (b & 0x7FL) << shift;
+ }
+ return i;
+ }
+
+ /** Call this if readString should read characters stored
+ * in the old modified UTF8 format (length in java chars
+ * and java's modified UTF8 encoding). This is used for
+ * indices written pre-2.4 See LUCENE-510 for details. */
+ public void setModifiedUTF8StringsMode() {
+ preUTF8Strings = true;
+ }
+
+ /** Reads a string.
+ * @see DataOutput#writeString(String)
+ */
+ public String readString() throws IOException {
+ if (preUTF8Strings)
+ return readModifiedUTF8String();
+ int length = readVInt();
+ if (bytes == null || length > bytes.length) {
+ bytes = new byte[ArrayUtil.oversize(length, 1)];
+ }
+ readBytes(bytes, 0, length);
+ return new String(bytes, 0, length, "UTF-8");
+ }
+
+ private String readModifiedUTF8String() throws IOException {
+ int length = readVInt();
+ if (chars == null || length > chars.length) {
+ chars = new char[ArrayUtil.oversize(length, RamUsageEstimator.NUM_BYTES_CHAR)];
+ }
+ readChars(chars, 0, length);
+ return new String(chars, 0, length);
+ }
+
+ /** Reads Lucene's old "modified UTF-8" encoded
+ * characters into an array.
+ * @param buffer the array to read characters into
+ * @param start the offset in the array to start storing characters
+ * @param length the number of characters to read
+ * @see DataOutput#writeChars(String,int,int)
+ * @deprecated -- please use readString or readBytes
+ * instead, and construct the string
+ * from those utf8 bytes
+ */
+ @Deprecated
+ public void readChars(char[] buffer, int start, int length)
+ throws IOException {
+ final int end = start + length;
+ for (int i = start; i < end; i++) {
+ byte b = readByte();
+ if ((b & 0x80) == 0)
+ buffer[i] = (char)(b & 0x7F);
+ else if ((b & 0xE0) != 0xE0) {
+ buffer[i] = (char)(((b & 0x1F) << 6)
+ | (readByte() & 0x3F));
+ } else {
+ buffer[i] = (char)(((b & 0x0F) << 12)
+ | ((readByte() & 0x3F) << 6)
+ | (readByte() & 0x3F));
+ }
+ }
+ }
+
+ /**
+ * Expert
+ *
+ * Similar to {@link #readChars(char[], int, int)} but does not do any conversion operations on the bytes it is reading in. It still
+ * has to invoke {@link #readByte()} just as {@link #readChars(char[], int, int)} does, but it does not need a buffer to store anything
+ * and it does not have to do any of the bitwise operations, since we don't actually care what is in the byte except to determine
+ * how many more bytes to read
+ * @param length The number of chars to read
+ * @deprecated this method operates on old "modified utf8" encoded
+ * strings
+ */
+ @Deprecated
+ public void skipChars(int length) throws IOException{
+ for (int i = 0; i < length; i++) {
+ byte b = readByte();
+ if ((b & 0x80) == 0){
+ //do nothing, we only need one byte
+ } else if ((b & 0xE0) != 0xE0) {
+ readByte();//read an additional byte
+ } else {
+ //read two additional bytes.
+ readByte();
+ readByte();
+ }
+ }
+ }
+
+ /** Returns a clone of this stream.
+ *
+ * <p>Clones of a stream access the same data, and are positioned at the same
+ * point as the stream they were cloned from.
+ *
+ * <p>Expert: Subclasses must ensure that clones may be positioned at
+ * different points in the input from each other and from the stream they
+ * were cloned from.
+ */
+ @Override
+ public Object clone() {
+ DataInput clone = null;
+ try {
+ clone = (DataInput)super.clone();
+ } catch (CloneNotSupportedException e) {}
+
+ clone.bytes = null;
+ clone.chars = null;
+
+ return clone;
+ }
+
+ public Map<String,String> readStringStringMap() throws IOException {
+ final Map<String,String> map = new HashMap<String,String>();
+ final int count = readInt();
+ for(int i=0;i<count;i++) {
+ final String key = readString();
+ final String val = readString();
+ map.put(key, val);
+ }
+
+ return map;
+ }
+}
Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataInput.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataOutput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataOutput.java?rev=929340&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataOutput.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataOutput.java Tue Mar 30 23:39:45 2010
@@ -0,0 +1,194 @@
+package org.apache.lucene.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.UnicodeUtil;
+
+/**
+ * Abstract base class for performing write operations of Lucene's low-level
+ * data types.
+ */
+public abstract class DataOutput {
+
+ private BytesRef utf8Result = new BytesRef(10);
+
+ /** Writes a single byte.
+ * @see IndexInput#readByte()
+ */
+ public abstract void writeByte(byte b) throws IOException;
+
+ /** Writes an array of bytes.
+ * @param b the bytes to write
+ * @param length the number of bytes to write
+ * @see DataInput#readBytes(byte[],int,int)
+ */
+ public void writeBytes(byte[] b, int length) throws IOException {
+ writeBytes(b, 0, length);
+ }
+
+ /** Writes an array of bytes.
+ * @param b the bytes to write
+ * @param offset the offset in the byte array
+ * @param length the number of bytes to write
+ * @see DataInput#readBytes(byte[],int,int)
+ */
+ public abstract void writeBytes(byte[] b, int offset, int length) throws IOException;
+
+ /** Writes an int as four bytes.
+ * @see DataInput#readInt()
+ */
+ public void writeInt(int i) throws IOException {
+ writeByte((byte)(i >> 24));
+ writeByte((byte)(i >> 16));
+ writeByte((byte)(i >> 8));
+ writeByte((byte) i);
+ }
+
+ /** Writes an int in a variable-length format. Writes between one and
+ * five bytes. Smaller values take fewer bytes. Negative numbers are not
+ * supported.
+ * @see DataInput#readVInt()
+ */
+ public void writeVInt(int i) throws IOException {
+ while ((i & ~0x7F) != 0) {
+ writeByte((byte)((i & 0x7f) | 0x80));
+ i >>>= 7;
+ }
+ writeByte((byte)i);
+ }
+
+ /** Writes a long as eight bytes.
+ * @see DataInput#readLong()
+ */
+ public void writeLong(long i) throws IOException {
+ writeInt((int) (i >> 32));
+ writeInt((int) i);
+ }
+
+ /** Writes an long in a variable-length format. Writes between one and five
+ * bytes. Smaller values take fewer bytes. Negative numbers are not
+ * supported.
+ * @see DataInput#readVLong()
+ */
+ public void writeVLong(long i) throws IOException {
+ while ((i & ~0x7F) != 0) {
+ writeByte((byte)((i & 0x7f) | 0x80));
+ i >>>= 7;
+ }
+ writeByte((byte)i);
+ }
+
+ /** Writes a string.
+ * @see DataInput#readString()
+ */
+ public void writeString(String s) throws IOException {
+ UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
+ writeVInt(utf8Result.length);
+ writeBytes(utf8Result.bytes, 0, utf8Result.length);
+ }
+
+ /** Writes a sub sequence of characters from s as the old
+ * format (modified UTF-8 encoded bytes).
+ * @param s the source of the characters
+ * @param start the first character in the sequence
+ * @param length the number of characters in the sequence
+ * @deprecated -- please pre-convert to utf8 bytes
+ * instead or use {@link #writeString}
+ */
+ @Deprecated
+ public void writeChars(String s, int start, int length)
+ throws IOException {
+ final int end = start + length;
+ for (int i = start; i < end; i++) {
+ final int code = s.charAt(i);
+ if (code >= 0x01 && code <= 0x7F)
+ writeByte((byte)code);
+ else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
+ writeByte((byte)(0xC0 | (code >> 6)));
+ writeByte((byte)(0x80 | (code & 0x3F)));
+ } else {
+ writeByte((byte)(0xE0 | (code >>> 12)));
+ writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
+ writeByte((byte)(0x80 | (code & 0x3F)));
+ }
+ }
+ }
+
+ /** Writes a sub sequence of characters from char[] as
+ * the old format (modified UTF-8 encoded bytes).
+ * @param s the source of the characters
+ * @param start the first character in the sequence
+ * @param length the number of characters in the sequence
+ * @deprecated -- please pre-convert to utf8 bytes instead or use {@link #writeString}
+ */
+ @Deprecated
+ public void writeChars(char[] s, int start, int length)
+ throws IOException {
+ final int end = start + length;
+ for (int i = start; i < end; i++) {
+ final int code = s[i];
+ if (code >= 0x01 && code <= 0x7F)
+ writeByte((byte)code);
+ else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
+ writeByte((byte)(0xC0 | (code >> 6)));
+ writeByte((byte)(0x80 | (code & 0x3F)));
+ } else {
+ writeByte((byte)(0xE0 | (code >>> 12)));
+ writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
+ writeByte((byte)(0x80 | (code & 0x3F)));
+ }
+ }
+ }
+
+ private static int COPY_BUFFER_SIZE = 16384;
+ private byte[] copyBuffer;
+
+ /** Copy numBytes bytes from input to ourself. */
+ public void copyBytes(DataInput input, long numBytes) throws IOException {
+ assert numBytes >= 0: "numBytes=" + numBytes;
+ long left = numBytes;
+ if (copyBuffer == null)
+ copyBuffer = new byte[COPY_BUFFER_SIZE];
+ while(left > 0) {
+ final int toCopy;
+ if (left > COPY_BUFFER_SIZE)
+ toCopy = COPY_BUFFER_SIZE;
+ else
+ toCopy = (int) left;
+ input.readBytes(copyBuffer, 0, toCopy);
+ writeBytes(copyBuffer, 0, toCopy);
+ left -= toCopy;
+ }
+ }
+
+ public void writeStringStringMap(Map<String,String> map) throws IOException {
+ if (map == null) {
+ writeInt(0);
+ } else {
+ writeInt(map.size());
+ for(final Map.Entry<String, String> entry: map.entrySet()) {
+ writeString(entry.getKey());
+ writeString(entry.getValue());
+ }
+ }
+ }
+}
Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/DataOutput.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java?rev=929340&r1=929339&r2=929340&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexInput.java Tue Mar 30 23:39:45 2010
@@ -17,194 +17,14 @@ package org.apache.lucene.store;
* limitations under the License.
*/
-import java.io.IOException;
import java.io.Closeable;
-import java.util.Map;
-import java.util.HashMap;
-
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.RamUsageEstimator;
+import java.io.IOException;
/** Abstract base class for input from a file in a {@link Directory}. A
* random-access input stream. Used for all Lucene index input operations.
* @see Directory
*/
-public abstract class IndexInput implements Cloneable,Closeable {
- private byte[] bytes; // used by readString()
- private char[] chars; // used by readModifiedUTF8String()
- private boolean preUTF8Strings; // true if we are reading old (modified UTF8) string format
-
- /** Reads and returns a single byte.
- * @see IndexOutput#writeByte(byte)
- */
- public abstract byte readByte() throws IOException;
-
- /** Reads a specified number of bytes into an array at the specified offset.
- * @param b the array to read bytes into
- * @param offset the offset in the array to start storing bytes
- * @param len the number of bytes to read
- * @see IndexOutput#writeBytes(byte[],int)
- */
- public abstract void readBytes(byte[] b, int offset, int len)
- throws IOException;
-
- /** Reads a specified number of bytes into an array at the
- * specified offset with control over whether the read
- * should be buffered (callers who have their own buffer
- * should pass in "false" for useBuffer). Currently only
- * {@link BufferedIndexInput} respects this parameter.
- * @param b the array to read bytes into
- * @param offset the offset in the array to start storing bytes
- * @param len the number of bytes to read
- * @param useBuffer set to false if the caller will handle
- * buffering.
- * @see IndexOutput#writeBytes(byte[],int)
- */
- public void readBytes(byte[] b, int offset, int len, boolean useBuffer)
- throws IOException
- {
- // Default to ignoring useBuffer entirely
- readBytes(b, offset, len);
- }
-
- /** Reads two bytes and returns a short.
- * @see IndexOutput#writeByte(byte)
- */
- public short readShort() throws IOException {
- return (short) (((readByte() & 0xFF) << 8) | (readByte() & 0xFF));
- }
-
- /** Reads four bytes and returns an int.
- * @see IndexOutput#writeInt(int)
- */
- public int readInt() throws IOException {
- return ((readByte() & 0xFF) << 24) | ((readByte() & 0xFF) << 16)
- | ((readByte() & 0xFF) << 8) | (readByte() & 0xFF);
- }
-
- /** Reads an int stored in variable-length format. Reads between one and
- * five bytes. Smaller values take fewer bytes. Negative numbers are not
- * supported.
- * @see IndexOutput#writeVInt(int)
- */
- public int readVInt() throws IOException {
- byte b = readByte();
- int i = b & 0x7F;
- for (int shift = 7; (b & 0x80) != 0; shift += 7) {
- b = readByte();
- i |= (b & 0x7F) << shift;
- }
- return i;
- }
-
- /** Reads eight bytes and returns a long.
- * @see IndexOutput#writeLong(long)
- */
- public long readLong() throws IOException {
- return (((long)readInt()) << 32) | (readInt() & 0xFFFFFFFFL);
- }
-
- /** Reads a long stored in variable-length format. Reads between one and
- * nine bytes. Smaller values take fewer bytes. Negative numbers are not
- * supported. */
- public long readVLong() throws IOException {
- byte b = readByte();
- long i = b & 0x7F;
- for (int shift = 7; (b & 0x80) != 0; shift += 7) {
- b = readByte();
- i |= (b & 0x7FL) << shift;
- }
- return i;
- }
-
- /** Call this if readString should read characters stored
- * in the old modified UTF8 format (length in java chars
- * and java's modified UTF8 encoding). This is used for
- * indices written pre-2.4 See LUCENE-510 for details. */
- public void setModifiedUTF8StringsMode() {
- preUTF8Strings = true;
- }
-
- /** Reads a string.
- * @see IndexOutput#writeString(String)
- */
- public String readString() throws IOException {
- if (preUTF8Strings)
- return readModifiedUTF8String();
- int length = readVInt();
- if (bytes == null || length > bytes.length) {
- bytes = new byte[ArrayUtil.oversize(length, 1)];
- }
- readBytes(bytes, 0, length);
- return new String(bytes, 0, length, "UTF-8");
- }
-
- private String readModifiedUTF8String() throws IOException {
- int length = readVInt();
- if (chars == null || length > chars.length) {
- chars = new char[ArrayUtil.oversize(length, RamUsageEstimator.NUM_BYTES_CHAR)];
- }
- readChars(chars, 0, length);
- return new String(chars, 0, length);
- }
-
- /** Reads Lucene's old "modified UTF-8" encoded
- * characters into an array.
- * @param buffer the array to read characters into
- * @param start the offset in the array to start storing characters
- * @param length the number of characters to read
- * @see IndexOutput#writeChars(String,int,int)
- * @deprecated -- please use readString or readBytes
- * instead, and construct the string
- * from those utf8 bytes
- */
- @Deprecated
- public void readChars(char[] buffer, int start, int length)
- throws IOException {
- final int end = start + length;
- for (int i = start; i < end; i++) {
- byte b = readByte();
- if ((b & 0x80) == 0)
- buffer[i] = (char)(b & 0x7F);
- else if ((b & 0xE0) != 0xE0) {
- buffer[i] = (char)(((b & 0x1F) << 6)
- | (readByte() & 0x3F));
- } else {
- buffer[i] = (char)(((b & 0x0F) << 12)
- | ((readByte() & 0x3F) << 6)
- | (readByte() & 0x3F));
- }
- }
- }
-
- /**
- * Expert
- *
- * Similar to {@link #readChars(char[], int, int)} but does not do any conversion operations on the bytes it is reading in. It still
- * has to invoke {@link #readByte()} just as {@link #readChars(char[], int, int)} does, but it does not need a buffer to store anything
- * and it does not have to do any of the bitwise operations, since we don't actually care what is in the byte except to determine
- * how many more bytes to read
- * @param length The number of chars to read
- * @deprecated this method operates on old "modified utf8" encoded
- * strings
- */
- @Deprecated
- public void skipChars(int length) throws IOException{
- for (int i = 0; i < length; i++) {
- byte b = readByte();
- if ((b & 0x80) == 0){
- //do nothing, we only need one byte
- } else if ((b & 0xE0) != 0xE0) {
- readByte();//read an additional byte
- } else {
- //read two additional bytes.
- readByte();
- readByte();
- }
- }
- }
-
-
+public abstract class IndexInput extends DataInput implements Cloneable,Closeable {
/** Closes the stream to further operations. */
public abstract void close() throws IOException;
@@ -221,38 +41,4 @@ public abstract class IndexInput impleme
/** The number of bytes in the file. */
public abstract long length();
-
- /** Returns a clone of this stream.
- *
- * <p>Clones of a stream access the same data, and are positioned at the same
- * point as the stream they were cloned from.
- *
- * <p>Expert: Subclasses must ensure that clones may be positioned at
- * different points in the input from each other and from the stream they
- * were cloned from.
- */
- @Override
- public Object clone() {
- IndexInput clone = null;
- try {
- clone = (IndexInput)super.clone();
- } catch (CloneNotSupportedException e) {}
-
- clone.bytes = null;
- clone.chars = null;
-
- return clone;
- }
-
- public Map<String,String> readStringStringMap() throws IOException {
- final Map<String,String> map = new HashMap<String,String>();
- final int count = readInt();
- for(int i=0;i<count;i++) {
- final String key = readString();
- final String val = readString();
- map.put(key, val);
- }
-
- return map;
- }
-}
+}
\ No newline at end of file
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java?rev=929340&r1=929339&r2=929340&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/IndexOutput.java Tue Mar 30 23:39:45 2010
@@ -17,170 +17,15 @@ package org.apache.lucene.store;
* limitations under the License.
*/
-import java.io.IOException;
import java.io.Closeable;
-import java.util.Map;
-
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.UnicodeUtil;
+import java.io.IOException;
/** Abstract base class for output to a file in a Directory. A random-access
* output stream. Used for all Lucene index output operations.
* @see Directory
* @see IndexInput
*/
-public abstract class IndexOutput implements Closeable {
-
- private BytesRef utf8Result = new BytesRef(10);
-
- /** Writes a single byte.
- * @see IndexInput#readByte()
- */
- public abstract void writeByte(byte b) throws IOException;
-
- /** Writes an array of bytes.
- * @param b the bytes to write
- * @param length the number of bytes to write
- * @see IndexInput#readBytes(byte[],int,int)
- */
- public void writeBytes(byte[] b, int length) throws IOException {
- writeBytes(b, 0, length);
- }
-
- /** Writes an array of bytes.
- * @param b the bytes to write
- * @param offset the offset in the byte array
- * @param length the number of bytes to write
- * @see IndexInput#readBytes(byte[],int,int)
- */
- public abstract void writeBytes(byte[] b, int offset, int length) throws IOException;
-
- /** Writes an int as four bytes.
- * @see IndexInput#readInt()
- */
- public void writeInt(int i) throws IOException {
- writeByte((byte)(i >> 24));
- writeByte((byte)(i >> 16));
- writeByte((byte)(i >> 8));
- writeByte((byte) i);
- }
-
- /** Writes an int in a variable-length format. Writes between one and
- * five bytes. Smaller values take fewer bytes. Negative numbers are not
- * supported.
- * @see IndexInput#readVInt()
- */
- public void writeVInt(int i) throws IOException {
- while ((i & ~0x7F) != 0) {
- writeByte((byte)((i & 0x7f) | 0x80));
- i >>>= 7;
- }
- writeByte((byte)i);
- }
-
- /** Writes a long as eight bytes.
- * @see IndexInput#readLong()
- */
- public void writeLong(long i) throws IOException {
- writeInt((int) (i >> 32));
- writeInt((int) i);
- }
-
- /** Writes an long in a variable-length format. Writes between one and five
- * bytes. Smaller values take fewer bytes. Negative numbers are not
- * supported.
- * @see IndexInput#readVLong()
- */
- public void writeVLong(long i) throws IOException {
- while ((i & ~0x7F) != 0) {
- writeByte((byte)((i & 0x7f) | 0x80));
- i >>>= 7;
- }
- writeByte((byte)i);
- }
-
- /** Writes a string.
- * @see IndexInput#readString()
- */
- public void writeString(String s) throws IOException {
- UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
- writeVInt(utf8Result.length);
- writeBytes(utf8Result.bytes, 0, utf8Result.length);
- }
-
- /** Writes a sub sequence of characters from s as the old
- * format (modified UTF-8 encoded bytes).
- * @param s the source of the characters
- * @param start the first character in the sequence
- * @param length the number of characters in the sequence
- * @deprecated -- please pre-convert to utf8 bytes
- * instead or use {@link #writeString}
- */
- @Deprecated
- public void writeChars(String s, int start, int length)
- throws IOException {
- final int end = start + length;
- for (int i = start; i < end; i++) {
- final int code = s.charAt(i);
- if (code >= 0x01 && code <= 0x7F)
- writeByte((byte)code);
- else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
- writeByte((byte)(0xC0 | (code >> 6)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- } else {
- writeByte((byte)(0xE0 | (code >>> 12)));
- writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- }
- }
- }
-
- /** Writes a sub sequence of characters from char[] as
- * the old format (modified UTF-8 encoded bytes).
- * @param s the source of the characters
- * @param start the first character in the sequence
- * @param length the number of characters in the sequence
- * @deprecated -- please pre-convert to utf8 bytes instead or use {@link #writeString}
- */
- @Deprecated
- public void writeChars(char[] s, int start, int length)
- throws IOException {
- final int end = start + length;
- for (int i = start; i < end; i++) {
- final int code = s[i];
- if (code >= 0x01 && code <= 0x7F)
- writeByte((byte)code);
- else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
- writeByte((byte)(0xC0 | (code >> 6)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- } else {
- writeByte((byte)(0xE0 | (code >>> 12)));
- writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
- writeByte((byte)(0x80 | (code & 0x3F)));
- }
- }
- }
-
- private static int COPY_BUFFER_SIZE = 16384;
- private byte[] copyBuffer;
-
- /** Copy numBytes bytes from input to ourself. */
- public void copyBytes(IndexInput input, long numBytes) throws IOException {
- assert numBytes >= 0: "numBytes=" + numBytes;
- long left = numBytes;
- if (copyBuffer == null)
- copyBuffer = new byte[COPY_BUFFER_SIZE];
- while(left > 0) {
- final int toCopy;
- if (left > COPY_BUFFER_SIZE)
- toCopy = COPY_BUFFER_SIZE;
- else
- toCopy = (int) left;
- input.readBytes(copyBuffer, 0, toCopy);
- writeBytes(copyBuffer, 0, toCopy);
- left -= toCopy;
- }
- }
+public abstract class IndexOutput extends DataOutput implements Closeable {
/** Forces any buffered output to be written. */
public abstract void flush() throws IOException;
@@ -213,16 +58,4 @@ public abstract class IndexOutput implem
* @param length file length
*/
public void setLength(long length) throws IOException {}
-
- public void writeStringStringMap(Map<String,String> map) throws IOException {
- if (map == null) {
- writeInt(0);
- } else {
- writeInt(map.size());
- for(final Map.Entry<String, String> entry: map.entrySet()) {
- writeString(entry.getKey());
- writeString(entry.getValue());
- }
- }
- }
}