You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by sy...@apache.org on 2014/09/16 00:47:06 UTC
[08/11] Skeleton porting of Lucene.Net.Misc
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/674f0cb9/src/Lucene.Net.Misc/Store/NativeUnixDirectory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Misc/Store/NativeUnixDirectory.cs b/src/Lucene.Net.Misc/Store/NativeUnixDirectory.cs
new file mode 100644
index 0000000..ad74aa1
--- /dev/null
+++ b/src/Lucene.Net.Misc/Store/NativeUnixDirectory.cs
@@ -0,0 +1,527 @@
+using System;
+using System.Diagnostics;
+
+namespace org.apache.lucene.store
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+ ignore
+ import java.nio.channels.FileChannel;
+
+ import org.apache.lucene.store.Directory; // javadoc
+ import org.apache.lucene.store.IOContext.Context;
+
+ // TODO
+ // - newer Linux kernel versions (after 2.6.29) have
+ // improved MADV_SEQUENTIAL (and hopefully also
+ // FADV_SEQUENTIAL) interaction with the buffer
+ // cache; we should explore using that instead of direct
+ // IO when context is merge
+
+ /// <summary>
+ /// A <seealso cref="Directory"/> implementation for all Unixes that uses
+ /// DIRECT I/O to bypass OS level IO caching during
+ /// merging. For all other cases (searching, writing) we delegate
+ /// to the provided Directory instance.
+ ///
+ /// <para>See <a
+ /// href="{@docRoot}/overview-summary.html#NativeUnixDirectory">Overview</a>
+ /// for more details.
+ ///
+ /// </para>
+ /// <para>To use this you must compile
+ /// NativePosixUtil.cpp (exposes Linux-specific APIs through
+ /// JNI) for your platform, by running <code>ant
+ /// build-native-unix</code>, and then putting the resulting
+ /// <code>libNativePosixUtil.so</code> (from
+ /// <code>lucene/build/native</code>) onto your dynamic
+ /// linker search path.
+ ///
+ /// </para>
+ /// <para><b>WARNING</b>: this code is very new and quite easily
+ /// could contain horrible bugs. For example, here's one
+ /// known issue: if you use seek in <code>IndexOutput</code>, and then
+ /// write more than one buffer's worth of bytes, then the
+ /// file will be wrong. Lucene does not do this today (only writes
+ /// small number of bytes after seek), but that may change.
+ ///
+ /// </para>
+ /// <para>This directory passes Solr and Lucene tests on Linux
+ /// and OS X; other Unixes should work but have not been
+ /// tested! Use at your own risk.
+ ///
+ /// @lucene.experimental
+ /// </para>
+ /// </summary>
+ public class NativeUnixDirectory extends FSDirectory
+ {
+
+ // TODO: this is OS dependent, but likely 512 is the LCD
+ private final static long ALIGN = 512;
+ private final static long ALIGN_NOT_MASK = ~(ALIGN - 1);
+
+ /// <summary>
+ /// Default buffer size before writing to disk (256 KB);
+ /// larger means less IO load but more RAM and direct
+ /// buffer storage space consumed during merging.
+ /// </summary>
+
+ public final static int DEFAULT_MERGE_BUFFER_SIZE = 262144;
+
+ /// <summary>
+ /// Default min expected merge size before direct IO is
+ /// used (10 MB):
+ /// </summary>
+ public final static long DEFAULT_MIN_BYTES_DIRECT = 10 * 1024 * 1024;
+
+ private final int mergeBufferSize;
+ private final long minBytesDirect;
+ private final Directory @delegate;
+
+ /// <summary>
+ /// Create a new NIOFSDirectory for the named location.
+ /// </summary>
+ /// <param name="path"> the path of the directory </param>
+ /// <param name="mergeBufferSize"> Size of buffer to use for
+ /// merging. See <seealso cref="#DEFAULT_MERGE_BUFFER_SIZE"/>. </param>
+ /// <param name="minBytesDirect"> Merges, or files to be opened for
+ /// reading, smaller than this will
+ /// not use direct IO. See {@link
+ /// #DEFAULT_MIN_BYTES_DIRECT} </param>
+ /// <param name="delegate"> fallback Directory for non-merges </param>
+ /// <exception cref="IOException"> If there is a low-level I/O error </exception>
+ public NativeUnixDirectory(File path, int mergeBufferSize, long minBytesDirect, Directory @delegate) throws IOException
+ {
+ base(path, @delegate.LockFactory);
+ if ((mergeBufferSize & ALIGN) != 0)
+ {
+ throw new System.ArgumentException("mergeBufferSize must be 0 mod " + ALIGN + " (got: " + mergeBufferSize + ")");
+ }
+ this.mergeBufferSize = mergeBufferSize;
+ this.minBytesDirect = minBytesDirect;
+ this.@delegate = @delegate;
+ }
+
+ /// <summary>
+ /// Create a new NIOFSDirectory for the named location.
+ /// </summary>
+ /// <param name="path"> the path of the directory </param>
+ /// <param name="delegate"> fallback Directory for non-merges </param>
+ /// <exception cref="IOException"> If there is a low-level I/O error </exception>
+ public NativeUnixDirectory(File path, Directory @delegate) throws IOException
+ {
+ this(path, DEFAULT_MERGE_BUFFER_SIZE, DEFAULT_MIN_BYTES_DIRECT, @delegate);
+ }
+
+ public IndexInput openInput(string name, IOContext context) throws IOException
+ {
+ ensureOpen();
+ if (context.context != Context.MERGE || context.mergeInfo.estimatedMergeBytes < minBytesDirect || fileLength(name) < minBytesDirect)
+ {
+ return @delegate.openInput(name, context);
+ }
+ else
+ {
+ return new NativeUnixIndexInput(new File(Directory, name), mergeBufferSize);
+ }
+ }
+
+ public IndexOutput createOutput(string name, IOContext context) throws IOException
+ {
+ ensureOpen();
+ if (context.context != Context.MERGE || context.mergeInfo.estimatedMergeBytes < minBytesDirect)
+ {
+ return @delegate.createOutput(name, context);
+ }
+ else
+ {
+ ensureCanWrite(name);
+ return new NativeUnixIndexOutput(new File(Directory, name), mergeBufferSize);
+ }
+ }
+
+ private final static class NativeUnixIndexOutput extends IndexOutput
+ {
+ private final ByteBuffer buffer;
+ private final FileOutputStream fos;
+ private final FileChannel channel;
+ private final int bufferSize;
+
+ //private final File path;
+
+ private int bufferPos;
+ private long filePos;
+ private long fileLength;
+ private bool isOpen;
+
+ public NativeUnixIndexOutput(File path, int bufferSize) throws IOException
+ {
+ //this.path = path;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final java.io.FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), false);
+ FileDescriptor fd = NativePosixUtil.open_direct(path.ToString(), false);
+ fos = new FileOutputStream(fd);
+ //fos = new FileOutputStream(path);
+ channel = fos.Channel;
+ buffer = ByteBuffer.allocateDirect(bufferSize);
+ this.bufferSize = bufferSize;
+ isOpen = true;
+ }
+
+ public void writeByte(sbyte b) throws IOException
+ {
+ Debug.Assert(bufferPos == buffer.position(), "bufferPos=" + bufferPos + " vs buffer.position()=" + buffer.position());
+ buffer.put(b);
+ if (++bufferPos == bufferSize)
+ {
+ dump();
+ }
+ }
+
+ public void writeBytes(sbyte[] src, int offset, int len) throws IOException
+ {
+ int toWrite = len;
+ while (true)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int left = bufferSize - bufferPos;
+ int left = bufferSize - bufferPos;
+ if (left <= toWrite)
+ {
+ buffer.put(src, offset, left);
+ toWrite -= left;
+ offset += left;
+ bufferPos = bufferSize;
+ dump();
+ }
+ else
+ {
+ buffer.put(src, offset, toWrite);
+ bufferPos += toWrite;
+ break;
+ }
+ }
+ }
+
+ //@Override
+ //public void setLength() throws IOException {
+ // TODO -- how to impl this? neither FOS nor
+ // FileChannel provides an API?
+ //}
+
+ public void flush()
+ {
+ // TODO -- I don't think this method is necessary?
+ }
+
+ private void dump() throws IOException
+ {
+ buffer.flip();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long limit = filePos + buffer.limit();
+ long limit = filePos + buffer.limit();
+ if (limit > fileLength)
+ {
+ // this dump extends the file
+ fileLength = limit;
+ }
+ else
+ {
+ // we had seek'd back & wrote some changes
+ }
+
+ // must always round to next block
+ buffer.limit((int)((buffer.limit() + ALIGN - 1) & ALIGN_NOT_MASK));
+
+ assert(buffer.limit() & ALIGN_NOT_MASK) == buffer.limit() : "limit=" + buffer.limit() + " vs " + (buffer.limit() & ALIGN_NOT_MASK);
+ assert(filePos & ALIGN_NOT_MASK) == filePos;
+ //System.out.println(Thread.currentThread().getName() + ": dump to " + filePos + " limit=" + buffer.limit() + " fos=" + fos);
+ channel.write(buffer, filePos);
+ filePos += bufferPos;
+ bufferPos = 0;
+ buffer.clear();
+ //System.out.println("dump: done");
+
+ // TODO: the case where we'd seek'd back, wrote an
+ // entire buffer, we must here read the next buffer;
+ // likely Lucene won't trip on this since we only
+ // write smallish amounts on seeking back
+ }
+
+ public long FilePointer
+ {
+ return filePos + bufferPos;
+ }
+
+ // TODO: seek is fragile at best; it can only properly
+ // handle seek & then change bytes that fit entirely
+ // within one buffer
+ public void seek(long pos) throws IOException
+ {
+ if (pos != FilePointer)
+ {
+ dump();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long alignedPos = pos & ALIGN_NOT_MASK;
+ long alignedPos = pos & ALIGN_NOT_MASK;
+ filePos = alignedPos;
+ int n = (int) NativePosixUtil.pread(fos.FD, filePos, buffer);
+ if (n < bufferSize)
+ {
+ buffer.limit(n);
+ }
+ //System.out.println("seek refill=" + n);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int delta = (int)(pos - alignedPos);
+ int delta = (int)(pos - alignedPos);
+ buffer.position(delta);
+ bufferPos = delta;
+ }
+ }
+
+ public long length()
+ {
+ return fileLength + bufferPos;
+ }
+
+ public long Checksum throws IOException
+ {
+ throw new System.NotSupportedException("this directory currently does not work at all!");
+ }
+
+ public void close() throws IOException
+ {
+ if (isOpen)
+ {
+ isOpen = false;
+ try
+ {
+ dump();
+ }
+ finally
+ {
+ try
+ {
+ //System.out.println("direct close set len=" + fileLength + " vs " + channel.size() + " path=" + path);
+ channel.truncate(fileLength);
+ //System.out.println(" now: " + channel.size());
+ }
+ finally
+ {
+ try
+ {
+ channel.close();
+ }
+ finally
+ {
+ fos.close();
+ //System.out.println(" final len=" + path.length());
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private final static class NativeUnixIndexInput extends IndexInput
+ {
+ private final ByteBuffer buffer;
+ private final FileInputStream fis;
+ private final FileChannel channel;
+ private final int bufferSize;
+
+ private bool isOpen;
+ private bool isClone;
+ private long filePos;
+ private int bufferPos;
+
+ public NativeUnixIndexInput(File path, int bufferSize) throws IOException
+ {
+ base("NativeUnixIndexInput(path=\"" + path.Path + "\")");
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final java.io.FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), true);
+ FileDescriptor fd = NativePosixUtil.open_direct(path.ToString(), true);
+ fis = new FileInputStream(fd);
+ channel = fis.Channel;
+ this.bufferSize = bufferSize;
+ buffer = ByteBuffer.allocateDirect(bufferSize);
+ isOpen = true;
+ isClone = false;
+ filePos = -bufferSize;
+ bufferPos = bufferSize;
+ //System.out.println("D open " + path + " this=" + this);
+ }
+
+ // for clone
+ public NativeUnixIndexInput(NativeUnixIndexInput other) throws IOException
+ {
+ base(other.ToString());
+ this.fis = null;
+ channel = other.channel;
+ this.bufferSize = other.bufferSize;
+ buffer = ByteBuffer.allocateDirect(bufferSize);
+ filePos = -bufferSize;
+ bufferPos = bufferSize;
+ isOpen = true;
+ isClone = true;
+ //System.out.println("D clone this=" + this);
+ seek(other.FilePointer);
+ }
+
+ public void close() throws IOException
+ {
+ if (isOpen && !isClone)
+ {
+ try
+ {
+ channel.close();
+ }
+ finally
+ {
+ if (!isClone)
+ {
+ fis.close();
+ }
+ }
+ }
+ }
+
+ public long FilePointer
+ {
+ return filePos + bufferPos;
+ }
+
+ public void seek(long pos) throws IOException
+ {
+ if (pos != FilePointer)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long alignedPos = pos & ALIGN_NOT_MASK;
+ long alignedPos = pos & ALIGN_NOT_MASK;
+ filePos = alignedPos - bufferSize;
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int delta = (int)(pos - alignedPos);
+ int delta = (int)(pos - alignedPos);
+ if (delta != 0)
+ {
+ refill();
+ buffer.position(delta);
+ bufferPos = delta;
+ }
+ else
+ {
+ // force refill on next read
+ bufferPos = bufferSize;
+ }
+ }
+ }
+
+ public long length()
+ {
+ try
+ {
+ return channel.size();
+ }
+ catch (IOException ioe)
+ {
+ throw new Exception("IOException during length(): " + this, ioe);
+ }
+ }
+
+ public sbyte readByte() throws IOException
+ {
+ // NOTE: we don't guard against EOF here... ie the
+ // "final" buffer will typically be filled to less
+ // than bufferSize
+ if (bufferPos == bufferSize)
+ {
+ refill();
+ }
+ Debug.Assert(bufferPos == buffer.position(), "bufferPos=" + bufferPos + " vs buffer.position()=" + buffer.position());
+ bufferPos++;
+ return buffer.get();
+ }
+
+ private void refill() throws IOException
+ {
+ buffer.clear();
+ filePos += bufferSize;
+ bufferPos = 0;
+ assert(filePos & ALIGN_NOT_MASK) == filePos : "filePos=" + filePos + " anded=" + (filePos & ALIGN_NOT_MASK);
+ //System.out.println("X refill filePos=" + filePos);
+ int n;
+ try
+ {
+ n = channel.read(buffer, filePos);
+ }
+ catch (IOException ioe)
+ {
+ throw new IOException(ioe.Message + ": " + this, ioe);
+ }
+ if (n < 0)
+ {
+ throw new EOFException("read past EOF: " + this);
+ }
+ buffer.rewind();
+ }
+
+ public void readBytes(sbyte[] dst, int offset, int len) throws IOException
+ {
+ int toRead = len;
+ //System.out.println("\nX readBytes len=" + len + " fp=" + getFilePointer() + " size=" + length() + " this=" + this);
+ while (true)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int left = bufferSize - bufferPos;
+ int left = bufferSize - bufferPos;
+ if (left < toRead)
+ {
+ //System.out.println(" copy " + left);
+ buffer.get(dst, offset, left);
+ toRead -= left;
+ offset += left;
+ refill();
+ }
+ else
+ {
+ //System.out.println(" copy " + toRead);
+ buffer.get(dst, offset, toRead);
+ bufferPos += toRead;
+ //System.out.println(" readBytes done");
+ break;
+ }
+ }
+ }
+
+ public NativeUnixIndexInput MemberwiseClone()
+ {
+ try
+ {
+ return new NativeUnixIndexInput(this);
+ }
+ catch (IOException ioe)
+ {
+ throw new Exception("IOException during clone: " + this, ioe);
+ }
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/674f0cb9/src/Lucene.Net.Misc/Store/WindowsDirectory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Misc/Store/WindowsDirectory.cs b/src/Lucene.Net.Misc/Store/WindowsDirectory.cs
new file mode 100644
index 0000000..324a0dd
--- /dev/null
+++ b/src/Lucene.Net.Misc/Store/WindowsDirectory.cs
@@ -0,0 +1,181 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace org.apache.lucene.store
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+
+
+ /// <summary>
+ /// Native <seealso cref="Directory"/> implementation for Microsoft Windows.
+ /// <para>
+ /// Steps:
+ /// <ol>
+ /// <li>Compile the source code to create WindowsDirectory.dll:
+ /// <blockquote>
+ /// c:\mingw\bin\g++ -Wall -D_JNI_IMPLEMENTATION_ -Wl,--kill-at
+ /// -I"%JAVA_HOME%\include" -I"%JAVA_HOME%\include\win32" -static-libgcc
+ /// -static-libstdc++ -shared WindowsDirectory.cpp -o WindowsDirectory.dll
+ /// </blockquote>
+ /// For 64-bit JREs, use mingw64, with the -m64 option.
+ /// <li>Put WindowsDirectory.dll into some directory in your windows PATH
+ /// <li>Open indexes with WindowsDirectory and use it.
+ /// </ol>
+ /// </para>
+ /// @lucene.experimental
+ /// </summary>
+ public class WindowsDirectory : FSDirectory
+ {
+ private const int DEFAULT_BUFFERSIZE = 4096; // default pgsize on ia32/amd64
+
+ static WindowsDirectory()
+ {
+//JAVA TO C# CONVERTER TODO TASK: The library is specified in the 'DllImport' attribute for .NET:
+// System.loadLibrary("WindowsDirectory");
+ }
+
+ /// <summary>
+ /// Create a new WindowsDirectory for the named location.
+ /// </summary>
+ /// <param name="path"> the path of the directory </param>
+ /// <param name="lockFactory"> the lock factory to use, or null for the default
+ /// (<seealso cref="NativeFSLockFactory"/>); </param>
+ /// <exception cref="IOException"> If there is a low-level I/O error </exception>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public WindowsDirectory(java.io.File path, LockFactory lockFactory) throws java.io.IOException
+ public WindowsDirectory(File path, LockFactory lockFactory) : base(path, lockFactory)
+ {
+ }
+
+ /// <summary>
+ /// Create a new WindowsDirectory for the named location and <seealso cref="NativeFSLockFactory"/>.
+ /// </summary>
+ /// <param name="path"> the path of the directory </param>
+ /// <exception cref="IOException"> If there is a low-level I/O error </exception>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public WindowsDirectory(java.io.File path) throws java.io.IOException
+ public WindowsDirectory(File path) : base(path, null)
+ {
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public IndexInput openInput(String name, IOContext context) throws java.io.IOException
+ public override IndexInput openInput(string name, IOContext context)
+ {
+ ensureOpen();
+ return new WindowsIndexInput(new File(Directory, name), Math.Max(BufferedIndexInput.bufferSize(context), DEFAULT_BUFFERSIZE));
+ }
+
+ internal class WindowsIndexInput : BufferedIndexInput
+ {
+ internal readonly long fd;
+ internal readonly long length_Renamed;
+ internal bool isClone;
+ internal bool isOpen;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public WindowsIndexInput(java.io.File file, int bufferSize) throws java.io.IOException
+ public WindowsIndexInput(File file, int bufferSize) : base("WindowsIndexInput(path=\"" + file.Path + "\")", bufferSize)
+ {
+ fd = WindowsDirectory.open(file.Path);
+ length_Renamed = WindowsDirectory.length(fd);
+ isOpen = true;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override protected void readInternal(byte[] b, int offset, int length) throws java.io.IOException
+ protected internal override void readInternal(sbyte[] b, int offset, int length)
+ {
+ int bytesRead;
+ try
+ {
+ bytesRead = WindowsDirectory.read(fd, b, offset, length, FilePointer);
+ }
+ catch (IOException ioe)
+ {
+ throw new IOException(ioe.Message + ": " + this, ioe);
+ }
+
+ if (bytesRead != length)
+ {
+ throw new EOFException("read past EOF: " + this);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override protected void seekInternal(long pos) throws java.io.IOException
+ protected internal override void seekInternal(long pos)
+ {
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public synchronized void close() throws java.io.IOException
+ public override void close()
+ {
+ lock (this)
+ {
+ // NOTE: we synchronize and track "isOpen" because Lucene sometimes closes IIs twice!
+ if (!isClone && isOpen)
+ {
+ WindowsDirectory.close(fd);
+ isOpen = false;
+ }
+ }
+ }
+
+ public override long length()
+ {
+ return length_Renamed;
+ }
+
+ public override WindowsIndexInput clone()
+ {
+ WindowsIndexInput clone = (WindowsIndexInput)base.clone();
+ clone.isClone = true;
+ return clone;
+ }
+ }
+
+ /// <summary>
+ /// Opens a handle to a file. </summary>
+//JAVA TO C# CONVERTER TODO TASK: Replace 'unknown' with the appropriate dll name:
+ [DllImport("unknown")]
+ private static extern long open(string filename);
+
+ /// <summary>
+ /// Reads data from a file at pos into bytes </summary>
+//JAVA TO C# CONVERTER TODO TASK: Replace 'unknown' with the appropriate dll name:
+ [DllImport("unknown")]
+ private static extern int read(long fd, sbyte[] bytes, int offset, int length, long pos);
+
+ /// <summary>
+ /// Closes a handle to a file </summary>
+//JAVA TO C# CONVERTER TODO TASK: Replace 'unknown' with the appropriate dll name:
+ [DllImport("unknown")]
+ private static extern void close(long fd);
+
+ /// <summary>
+ /// Returns the length of a file </summary>
+//JAVA TO C# CONVERTER TODO TASK: Replace 'unknown' with the appropriate dll name:
+ [DllImport("unknown")]
+ private static extern long length(long fd);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/674f0cb9/src/Lucene.Net.Misc/Util/Fst/ListOfOutputs.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Misc/Util/Fst/ListOfOutputs.cs b/src/Lucene.Net.Misc/Util/Fst/ListOfOutputs.cs
new file mode 100644
index 0000000..e4660a6
--- /dev/null
+++ b/src/Lucene.Net.Misc/Util/Fst/ListOfOutputs.cs
@@ -0,0 +1,246 @@
+using System.Diagnostics;
+using System.Collections;
+using System.Collections.Generic;
+using System.Text;
+
+namespace org.apache.lucene.util.fst
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using DataInput = org.apache.lucene.store.DataInput;
+ using DataOutput = org.apache.lucene.store.DataOutput;
+
+ /// <summary>
+ /// Wraps another Outputs implementation and encodes one or
+ /// more of its output values. You can use this when a single
+ /// input may need to map to more than one output,
+ /// maintaining order: pass the same input with a different
+ /// output by calling <seealso cref="Builder#add(IntsRef,Object)"/> multiple
+ /// times. The builder will then combine the outputs using
+ /// the <seealso cref="Outputs#merge(Object,Object)"/> method.
+ ///
+ /// <para>The resulting FST may not be minimal when an input has
+ /// more than one output, as this requires pushing all
+ /// multi-output values to a final state.
+ ///
+ /// </para>
+ /// <para>NOTE: the only way to create multiple outputs is to
+ /// add the same input to the FST multiple times in a row. This is
+ /// how the FST maps a single input to multiple outputs (e.g. you
+ /// cannot pass a List<Object> to <seealso cref="Builder#add"/>). If
+ /// your outputs are longs, and you need at most 2, then use
+ /// <seealso cref="UpToTwoPositiveIntOutputs"/> instead since it stores
+ /// the outputs more compactly (by stealing a bit from each
+ /// long value).
+ ///
+ /// </para>
+ /// <para>NOTE: this cannot wrap itself (ie you cannot make an
+ /// FST with List<List<Object>> outputs using this).
+ ///
+ /// @lucene.experimental
+ /// </para>
+ /// </summary>
+
+
+ // NOTE: i think we could get a more compact FST if, instead
+ // of adding the same input multiple times with a different
+ // output each time, we added it only once with a
+ // pre-constructed List<T> output. This way the "multiple
+ // values" is fully opaque to the Builder/FST. It would
+ // require implementing the full algebra using set
+ // arithmetic (I think?); maybe SetOfOutputs is a good name.
+
+//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
+//ORIGINAL LINE: @SuppressWarnings("unchecked") public final class ListOfOutputs<T> extends Outputs<Object>
+ public sealed class ListOfOutputs<T> : Outputs<object>
+ {
+
+ private readonly Outputs<T> outputs;
+
+ public ListOfOutputs(Outputs<T> outputs)
+ {
+ this.outputs = outputs;
+ }
+
+ public override object common(object output1, object output2)
+ {
+ // These will never be a list:
+ return outputs.common((T) output1, (T) output2);
+ }
+
+ public override object subtract(object @object, object inc)
+ {
+ // These will never be a list:
+ return outputs.subtract((T) @object, (T) inc);
+ }
+
+ public override object add(object prefix, object output)
+ {
+ Debug.Assert(!(prefix is IList));
+ if (!(output is IList))
+ {
+ return outputs.add((T) prefix, (T) output);
+ }
+ else
+ {
+ IList<T> outputList = (IList<T>) output;
+ IList<T> addedList = new List<T>(outputList.Count);
+ foreach (T _output in outputList)
+ {
+ addedList.Add(outputs.add((T) prefix, _output));
+ }
+ return addedList;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void write(Object output, org.apache.lucene.store.DataOutput out) throws java.io.IOException
+ public override void write(object output, DataOutput @out)
+ {
+ Debug.Assert(!(output is IList));
+ outputs.write((T) output, @out);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void writeFinalOutput(Object output, org.apache.lucene.store.DataOutput out) throws java.io.IOException
+ public override void writeFinalOutput(object output, DataOutput @out)
+ {
+ if (!(output is IList))
+ {
+ @out.writeVInt(1);
+ outputs.write((T) output, @out);
+ }
+ else
+ {
+ IList<T> outputList = (IList<T>) output;
+ @out.writeVInt(outputList.Count);
+ foreach (T eachOutput in outputList)
+ {
+ outputs.write(eachOutput, @out);
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public Object read(org.apache.lucene.store.DataInput in) throws java.io.IOException
+ public override object read(DataInput @in)
+ {
+ return outputs.read(@in);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public Object readFinalOutput(org.apache.lucene.store.DataInput in) throws java.io.IOException
+ public override object readFinalOutput(DataInput @in)
+ {
+ int count = @in.readVInt();
+ if (count == 1)
+ {
+ return outputs.read(@in);
+ }
+ else
+ {
+ IList<T> outputList = new List<T>(count);
+ for (int i = 0;i < count;i++)
+ {
+ outputList.Add(outputs.read(@in));
+ }
+ return outputList;
+ }
+ }
+
+ public override object NoOutput
+ {
+ get
+ {
+ return outputs.NoOutput;
+ }
+ }
+
+ public override string outputToString(object output)
+ {
+ if (!(output is IList))
+ {
+ return outputs.outputToString((T) output);
+ }
+ else
+ {
+ IList<T> outputList = (IList<T>) output;
+
+ StringBuilder b = new StringBuilder();
+ b.Append('[');
+
+ for (int i = 0;i < outputList.Count;i++)
+ {
+ if (i > 0)
+ {
+ b.Append(", ");
+ }
+ b.Append(outputs.outputToString(outputList[i]));
+ }
+ b.Append(']');
+ return b.ToString();
+ }
+ }
+
+ public override object merge(object first, object second)
+ {
+ IList<T> outputList = new List<T>();
+ if (!(first is IList))
+ {
+ outputList.Add((T) first);
+ }
+ else
+ {
+ outputList.AddRange((IList<T>) first);
+ }
+ if (!(second is IList))
+ {
+ outputList.Add((T) second);
+ }
+ else
+ {
+ outputList.AddRange((IList<T>) second);
+ }
+ //System.out.println("MERGE: now " + outputList.size() + " first=" + outputToString(first) + " second=" + outputToString(second));
+ //System.out.println(" return " + outputToString(outputList));
+ return outputList;
+ }
+
+ public override string ToString()
+ {
+ return "OneOrMoreOutputs(" + outputs + ")";
+ }
+
+ public IList<T> asList(object output)
+ {
+ if (!(output is IList))
+ {
+ IList<T> result = new List<T>(1);
+ result.Add((T) output);
+ return result;
+ }
+ else
+ {
+ return (IList<T>) output;
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/674f0cb9/src/Lucene.Net.Misc/Util/Fst/UpToTwoPositiveIntOutputs.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Misc/Util/Fst/UpToTwoPositiveIntOutputs.cs b/src/Lucene.Net.Misc/Util/Fst/UpToTwoPositiveIntOutputs.cs
new file mode 100644
index 0000000..8302127
--- /dev/null
+++ b/src/Lucene.Net.Misc/Util/Fst/UpToTwoPositiveIntOutputs.cs
@@ -0,0 +1,328 @@
+using System;
+using System.Diagnostics;
+
+namespace org.apache.lucene.util.fst
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using DataInput = org.apache.lucene.store.DataInput;
+ using DataOutput = org.apache.lucene.store.DataOutput;
+
+ /// <summary>
+ /// An FST <seealso cref="Outputs"/> implementation where each output
+ /// is one or two non-negative long values. If it's a
+ /// single output, Long is returned; else, TwoLongs. Order
+ /// is preserved in the TwoLongs case, ie .first is the first
+ /// input/output added to Builder, and .second is the
+ /// second. You cannot store 0 output with this (that's
+ /// reserved to mean "no output")!
+ ///
+ /// <para>NOTE: the only way to create a TwoLongs output is to
+ /// add the same input to the FST twice in a row. This is
+ /// how the FST maps a single input to two outputs (e.g. you
+ /// cannot pass a TwoLongs to <seealso cref="Builder#add"/>. If you
+ /// need more than two then use <seealso cref="ListOfOutputs"/>, but if
+ /// you only have at most 2 then this implementation will
+ /// require fewer bytes as it steals one bit from each long
+ /// value.
+ ///
+ /// </para>
+ /// <para>NOTE: the resulting FST is not guaranteed to be minimal!
+ /// See <seealso cref="Builder"/>.
+ ///
+ /// @lucene.experimental
+ /// </para>
+ /// </summary>
+
+ public sealed class UpToTwoPositiveIntOutputs : Outputs<object>
+ {
+
+ /// <summary>
+ /// Holds two long outputs. </summary>
+ public sealed class TwoLongs
+ {
+ public readonly long first;
+ public readonly long second;
+
+ public TwoLongs(long first, long second)
+ {
+ this.first = first;
+ this.second = second;
+ Debug.Assert(first >= 0);
+ Debug.Assert(second >= 0);
+ }
+
+ public override string ToString()
+ {
+ return "TwoLongs:" + first + "," + second;
+ }
+
+ public override bool Equals(object _other)
+ {
+ if (_other is TwoLongs)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TwoLongs other = (TwoLongs) _other;
+ TwoLongs other = (TwoLongs) _other;
+ return first == other.first && second == other.second;
+ }
+ else
+ {
+ return false;
+ }
+ }
+
+ public override int GetHashCode()
+ {
+ return (int)((first ^ ((long)((ulong)first >> 32))) ^ (second ^ (second >> 32)));
+ }
+ }
+
+ private static readonly long? NO_OUTPUT = new long?(0);
+
+ private readonly bool doShare;
+
+ private static readonly UpToTwoPositiveIntOutputs singletonShare = new UpToTwoPositiveIntOutputs(true);
+ private static readonly UpToTwoPositiveIntOutputs singletonNoShare = new UpToTwoPositiveIntOutputs(false);
+
+ private UpToTwoPositiveIntOutputs(bool doShare)
+ {
+ this.doShare = doShare;
+ }
+
+ public static UpToTwoPositiveIntOutputs getSingleton(bool doShare)
+ {
+ return doShare ? singletonShare : singletonNoShare;
+ }
+
+ public long? get(long v)
+ {
+ if (v == 0)
+ {
+ return NO_OUTPUT;
+ }
+ else
+ {
+ return Convert.ToInt64(v);
+ }
+ }
+
+ public TwoLongs get(long first, long second)
+ {
+ return new TwoLongs(first, second);
+ }
+
+ public override long? common(object _output1, object _output2)
+ {
+ Debug.Assert(valid(_output1, false));
+ Debug.Assert(valid(_output2, false));
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final Long output1 = (Long) _output1;
+ long? output1 = (long?) _output1;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final Long output2 = (Long) _output2;
+ long? output2 = (long?) _output2;
+ if (output1 == NO_OUTPUT || output2 == NO_OUTPUT)
+ {
+ return NO_OUTPUT;
+ }
+ else if (doShare)
+ {
+ Debug.Assert(output1 > 0);
+ Debug.Assert(output2 > 0);
+ return Math.Min(output1, output2);
+ }
+ else if (output1.Equals(output2))
+ {
+ return output1;
+ }
+ else
+ {
+ return NO_OUTPUT;
+ }
+ }
+
+ public override long? subtract(object _output, object _inc)
+ {
+ Debug.Assert(valid(_output, false));
+ Debug.Assert(valid(_inc, false));
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final Long output = (Long) _output;
+ long? output = (long?) _output;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final Long inc = (Long) _inc;
+ long? inc = (long?) _inc;
+ Debug.Assert(output >= inc);
+
+ if (inc == NO_OUTPUT)
+ {
+ return output;
+ }
+ else if (output.Equals(inc))
+ {
+ return NO_OUTPUT;
+ }
+ else
+ {
+ return output - inc;
+ }
+ }
+
+ public override object add(object _prefix, object _output)
+ {
+ Debug.Assert(valid(_prefix, false));
+ Debug.Assert(valid(_output, true));
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final Long prefix = (Long) _prefix;
+ long? prefix = (long?) _prefix;
+ if (_output is long?)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final Long output = (Long) _output;
+ long? output = (long?) _output;
+ if (prefix == NO_OUTPUT)
+ {
+ return output;
+ }
+ else if (output == NO_OUTPUT)
+ {
+ return prefix;
+ }
+ else
+ {
+ return prefix + output;
+ }
+ }
+ else
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TwoLongs output = (TwoLongs) _output;
+ TwoLongs output = (TwoLongs) _output;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long v = prefix;
+ long v = prefix.Value;
+ return new TwoLongs(output.first + v, output.second + v);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void write(Object _output, org.apache.lucene.store.DataOutput out) throws java.io.IOException
+ public override void write(object _output, DataOutput @out)
+ {
+ Debug.Assert(valid(_output, true));
+ if (_output is long?)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final Long output = (Long) _output;
+ long? output = (long?) _output;
+ @out.writeVLong(output << 1);
+ }
+ else
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TwoLongs output = (TwoLongs) _output;
+ TwoLongs output = (TwoLongs) _output;
+ @out.writeVLong((output.first << 1) | 1);
+ @out.writeVLong(output.second);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public Object read(org.apache.lucene.store.DataInput in) throws java.io.IOException
+ public override object read(DataInput @in)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long code = in.readVLong();
+ long code = @in.readVLong();
+ if ((code & 1) == 0)
+ {
+ // single long
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long v = code >>> 1;
+ long v = (long)((ulong)code >> 1);
+ if (v == 0)
+ {
+ return NO_OUTPUT;
+ }
+ else
+ {
+ return Convert.ToInt64(v);
+ }
+ }
+ else
+ {
+ // two longs
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long first = code >>> 1;
+ long first = (long)((ulong)code >> 1);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final long second = in.readVLong();
+ long second = @in.readVLong();
+ return new TwoLongs(first, second);
+ }
+ }
+
+ private bool valid(long? o)
+ {
+ Debug.Assert(o != null);
+ Debug.Assert(o is long?);
+ Debug.Assert(o == NO_OUTPUT || o > 0);
+ return true;
+ }
+
+ // Used only by assert
+ private bool valid(object _o, bool allowDouble)
+ {
+ if (!allowDouble)
+ {
+ Debug.Assert(_o is long?);
+ return valid((long?) _o);
+ }
+ else if (_o is TwoLongs)
+ {
+ return true;
+ }
+ else
+ {
+ return valid((long?) _o);
+ }
+ }
+
+ public override object NoOutput
+ {
+ get
+ {
+ return NO_OUTPUT;
+ }
+ }
+
+ public override string outputToString(object output)
+ {
+ return output.ToString();
+ }
+
+ public override object merge(object first, object second)
+ {
+ Debug.Assert(valid(first, false));
+ Debug.Assert(valid(second, false));
+ return new TwoLongs((long?) first, (long?) second);
+ }
+ }
+
+}
\ No newline at end of file