You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by sy...@apache.org on 2016/03/25 09:19:31 UTC
lucenenet git commit: LUCENET-555 Removing dependency on external zip
library.
Repository: lucenenet
Updated Branches:
refs/heads/master c6d7bec47 -> e218ff63d
LUCENET-555 Removing dependency on external zip library.
Adding ignore for project.lock.json files.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/e218ff63
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/e218ff63
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/e218ff63
Branch: refs/heads/master
Commit: e218ff63d0b22cbe39543986f0382f241cf47c24
Parents: c6d7bec
Author: Elizabeth Maher <el...@microsoft.com>
Authored: Thu Mar 24 15:42:32 2016 -0700
Committer: Elizabeth Maher <el...@microsoft.com>
Committed: Thu Mar 24 19:07:22 2016 -0700
----------------------------------------------------------------------
.gitignore | 3 +-
.../Codecs/Compressing/CompressionMode.cs | 106 +++++++------------
.../Document/CompressionTools.cs | 80 +++++---------
src/Lucene.Net.Core/Lucene.Net.csproj | 3 -
src/Lucene.Net.TestFramework/Util/TestUtil.cs | 96 -----------------
5 files changed, 70 insertions(+), 218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e218ff63/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index a1ea325..27e2d1b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@ obj
*.userprefs
*.pidb
*.ide
+*.project.lock.json
test-results
build/artifacts
build/bin
@@ -31,4 +32,4 @@ bin/
obj/
doc/
src/demo/
-packages/
\ No newline at end of file
+packages/
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e218ff63/src/Lucene.Net.Core/Codecs/Compressing/CompressionMode.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/Compressing/CompressionMode.cs b/src/Lucene.Net.Core/Codecs/Compressing/CompressionMode.cs
index be238b3..f319483 100644
--- a/src/Lucene.Net.Core/Codecs/Compressing/CompressionMode.cs
+++ b/src/Lucene.Net.Core/Codecs/Compressing/CompressionMode.cs
@@ -1,4 +1,6 @@
using System.Diagnostics;
+using System.IO;
+using System.IO.Compression;
namespace Lucene.Net.Codecs.Compressing
{
@@ -81,7 +83,7 @@ namespace Lucene.Net.Codecs.Compressing
public override Compressor NewCompressor()
{
- return new DeflateCompressor(Deflater.BEST_COMPRESSION);
+ return new DeflateCompressor(System.IO.Compression.CompressionLevel.Optimal);
}
public override Decompressor NewDecompressor()
@@ -204,16 +206,12 @@ namespace Lucene.Net.Codecs.Compressing
private sealed class DeflateDecompressor : Decompressor
{
- internal readonly Inflater decompressor;
- internal byte[] Compressed;
internal DeflateDecompressor()
{
- decompressor = SharpZipLib.CreateInflater();
- Compressed = new byte[0];
}
- public override void Decompress(DataInput @in, int originalLength, int offset, int length, BytesRef bytes)
+ public override void Decompress(DataInput input, int originalLength, int offset, int length, BytesRef bytes)
{
Debug.Assert(offset + length <= originalLength);
if (length == 0)
@@ -221,45 +219,31 @@ namespace Lucene.Net.Codecs.Compressing
bytes.Length = 0;
return;
}
- int compressedLength = @in.ReadVInt();
- if (compressedLength > Compressed.Length)
- {
- Compressed = new byte[ArrayUtil.Oversize(compressedLength, 1)];
- }
- @in.ReadBytes(Compressed, 0, compressedLength);
- decompressor.Reset();
- decompressor.SetInput(Compressed, 0, compressedLength);
+ byte[] compressedBytes = new byte[input.ReadVInt()];
+ input.ReadBytes(compressedBytes, 0, compressedBytes.Length);
+ byte[] decompressedBytes = null;
- bytes.Offset = bytes.Length = 0;
- while (true)
+ using (MemoryStream decompressedStream = new MemoryStream())
{
- int count;
- try
- {
- int remaining = bytes.Bytes.Length - bytes.Length;
- count = decompressor.Inflate((byte[])(Array)(bytes.Bytes), bytes.Length, remaining);
- }
- catch (System.FormatException e)
- {
- throw new System.IO.IOException("See inner", e);
- }
- bytes.Length += count;
- if (decompressor.IsFinished)
- {
- break;
- }
- else
+ using (MemoryStream compressedStream = new MemoryStream(compressedBytes))
{
- bytes.Bytes = ArrayUtil.Grow(bytes.Bytes);
+ using (DeflateStream dStream = new DeflateStream(compressedStream, System.IO.Compression.CompressionMode.Decompress))
+ {
+ dStream.CopyTo(decompressedStream);
+ }
}
+ decompressedBytes = decompressedStream.ToArray();
}
- if (bytes.Length != originalLength)
+
+ if (decompressedBytes.Length != originalLength)
{
- throw new CorruptIndexException("Lengths mismatch: " + bytes.Length + " != " + originalLength + " (resource=" + @in + ")");
+ throw new CorruptIndexException($"Length mismatch: {decompressedBytes.Length} != {originalLength} (resource={input})");
}
+
+ bytes.Bytes = decompressedBytes;
bytes.Offset = offset;
- bytes.Length = length;
+ bytes.Length = length;
}
public override object Clone()
@@ -270,48 +254,36 @@ namespace Lucene.Net.Codecs.Compressing
private class DeflateCompressor : Compressor
{
- private readonly Deflater Compressor;
- private byte[] Compressed;
-
- internal DeflateCompressor(int level)
+ private CompressionLevel compressionLevel;
+ internal DeflateCompressor(CompressionLevel level)
{
- Compressor = SharpZipLib.CreateDeflater();
- Compressed = new byte[64];
+ compressionLevel = level;
}
- public override void Compress(byte[] bytes, int off, int len, DataOutput @out)
+ public override void Compress(byte[] bytes, int off, int len, DataOutput output)
{
- Compressor.Reset();
- Compressor.SetInput((byte[])(Array)bytes, off, len);
- Compressor.Finish();
+ byte[] resultArray = null;
+ using (MemoryStream compressionMemoryStream = new MemoryStream())
+ {
+ using (DeflateStream deflateStream = new DeflateStream(compressionMemoryStream, compressionLevel))
+ {
+ deflateStream.Write(bytes, off, len);
+ }
+ resultArray = compressionMemoryStream.ToArray();
+ }
- if (Compressor.NeedsInput)
+ if (resultArray.Length == 0)
{
- // no output
Debug.Assert(len == 0, len.ToString());
- @out.WriteVInt(0);
+ output.WriteVInt(0);
return;
}
-
- int totalCount = 0;
- for (; ; )
+ else
{
- int count = Compressor.Deflate(Compressed, totalCount, Compressed.Length - totalCount);
- totalCount += count;
- Debug.Assert(totalCount <= Compressed.Length);
- if (Compressor.IsFinished)
- {
- break;
- }
- else
- {
- Compressed = ArrayUtil.Grow(Compressed);
- }
+ output.WriteVInt(resultArray.Length);
+ output.WriteBytes(resultArray, resultArray.Length);
}
-
- @out.WriteVInt(totalCount);
- @out.WriteBytes(Compressed, totalCount);
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e218ff63/src/Lucene.Net.Core/Document/CompressionTools.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Document/CompressionTools.cs b/src/Lucene.Net.Core/Document/CompressionTools.cs
index d5e369d..b7e72b6 100644
--- a/src/Lucene.Net.Core/Document/CompressionTools.cs
+++ b/src/Lucene.Net.Core/Document/CompressionTools.cs
@@ -1,4 +1,6 @@
using System;
+using System.IO;
+using System.IO.Compression;
using Lucene.Net.Support;
using Lucene.Net.Util;
@@ -37,67 +39,50 @@ namespace Lucene.Net.Documents
/// <summary>
/// Compresses the specified byte range using the
- /// specified compressionLevel (constants are defined in
- /// java.util.zip.Deflater).
+ /// specified compressionLevel
/// </summary>
- public static byte[] Compress(byte[] value, int offset, int length, int compressionLevel)
+ public static byte[] Compress(byte[] value, int offset, int length, CompressionLevel compressionLevel)
{
- /* Create an expandable byte array to hold the compressed data.
- * You cannot use an array that's the same size as the orginal because
- * there is no guarantee that the compressed data will be smaller than
- * the uncompressed data. */
- var bos = new ByteArrayOutputStream(length);
-
- Deflater compressor = SharpZipLib.CreateDeflater();
-
- try
+ byte[] resultArray = null;
+ using (MemoryStream compressionMemoryStream = new MemoryStream())
{
- compressor.SetLevel(compressionLevel);
- compressor.SetInput(value, offset, length);
- compressor.Finish();
-
- // Compress the data
- var buf = new byte[1024];
- while (!compressor.IsFinished)
+ using (DeflateStream deflateStream = new DeflateStream(compressionMemoryStream, compressionLevel))
{
- int count = compressor.Deflate(buf);
- bos.Write(buf, 0, count);
+
+ deflateStream.Write(value, offset, length);
}
+ resultArray = compressionMemoryStream.ToArray();
}
- finally
- {
- }
-
- return bos.ToArray();
+ return resultArray;
}
/// <summary>
- /// Compresses the specified byte range, with default BEST_COMPRESSION level </summary>
+ /// Compresses the specified byte range, with default Optimal level
+ /// </summary>
public static byte[] Compress(byte[] value, int offset, int length)
{
- return Compress(value, offset, length, Deflater.BEST_COMPRESSION);
+ return Compress(value, offset, length, CompressionLevel.Optimal);
}
/// <summary>
- /// Compresses all bytes in the array, with default BEST_COMPRESSION level </summary>
+ /// Compresses all bytes in the array, with default Optimal level </summary>
public static byte[] Compress(byte[] value)
{
- return Compress(value, 0, value.Length, Deflater.BEST_COMPRESSION);
+ return Compress(value, 0, value.Length, CompressionLevel.Optimal);
}
/// <summary>
/// Compresses the String value, with default BEST_COMPRESSION level </summary>
public static byte[] CompressString(string value)
{
- return CompressString(value, Deflater.BEST_COMPRESSION);
+ return CompressString(value, CompressionLevel.Optimal);
}
/// <summary>
/// Compresses the String value using the specified
- /// compressionLevel (constants are defined in
- /// java.util.zip.Deflater).
+ /// compressionLevel.
/// </summary>
- public static byte[] CompressString(string value, int compressionLevel)
+ public static byte[] CompressString(string value, CompressionLevel compressionLevel)
{
var result = new BytesRef();
UnicodeUtil.UTF16toUTF8(value.ToCharArray(), 0, value.Length, result);
@@ -128,28 +113,21 @@ namespace Lucene.Net.Documents
/// </summary>
public static byte[] Decompress(byte[] value, int offset, int length)
{
- // Create an expandable byte array to hold the decompressed data
- var bos = new ByteArrayOutputStream(length);
-
- Inflater decompressor = SharpZipLib.CreateInflater();
+ byte[] decompressedBytes = null;
- try
+ using (MemoryStream decompressedStream = new MemoryStream())
{
- decompressor.SetInput(value);
-
- // Decompress the data
- var buf = new byte[1024];
- while (!decompressor.IsFinished)
+ using (MemoryStream compressedStream = new MemoryStream(value))
{
- int count = decompressor.Inflate(buf);
- bos.Write(buf, 0, count);
+ using (DeflateStream dStream = new DeflateStream(compressedStream, CompressionMode.Decompress))
+ {
+ dStream.CopyTo(decompressedStream);
+ }
}
- }
- finally
- {
+ decompressedBytes = decompressedStream.ToArray();
}
- return bos.ToArray();
+ return decompressedBytes;
}
/// <summary>
@@ -182,4 +160,4 @@ namespace Lucene.Net.Documents
return DecompressString(bytes.Bytes, bytes.Offset, bytes.Length);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e218ff63/src/Lucene.Net.Core/Lucene.Net.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Lucene.Net.csproj b/src/Lucene.Net.Core/Lucene.Net.csproj
index 7d3e2c1..5d573a0 100644
--- a/src/Lucene.Net.Core/Lucene.Net.csproj
+++ b/src/Lucene.Net.Core/Lucene.Net.csproj
@@ -40,9 +40,6 @@
<StartupObject />
</PropertyGroup>
<ItemGroup>
- <Reference Include="ICSharpCode.SharpZipLib">
- <HintPath>..\..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath>
- </Reference>
<Reference Include="Microsoft.CSharp" />
<Reference Include="System" />
<Reference Include="System.Collections.Immutable">
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e218ff63/src/Lucene.Net.TestFramework/Util/TestUtil.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.TestFramework/Util/TestUtil.cs b/src/Lucene.Net.TestFramework/Util/TestUtil.cs
index 7250cdb..1afcf67 100644
--- a/src/Lucene.Net.TestFramework/Util/TestUtil.cs
+++ b/src/Lucene.Net.TestFramework/Util/TestUtil.cs
@@ -1,5 +1,4 @@
using System.Numerics;
-using ICSharpCode.SharpZipLib.Zip;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using System;
@@ -124,101 +123,6 @@ namespace Lucene.Net.Util
return unremoved;
}
- /// <summary>
- /// Convenience method unzipping zipName into destDir, cleaning up
- /// destDir first.
- /// </summary>
- public static void Unzip(FileInfo zipName, DirectoryInfo destDir)
- {
- System.IO.Directory.Delete(destDir.FullName, true);
- destDir.CreateSubdirectory(destDir.FullName);// mkdir();
-
- using (ZipInputStream s = new ZipInputStream(File.OpenRead(zipName.FullName)))
- {
- ZipEntry entry;
- while ((entry = s.GetNextEntry()) != null)
- {
- string directoryName = Path.GetDirectoryName(entry.Name); //Directory where file is stored
- string fileName = Path.GetFileName(entry.Name); //Name of file (if it is one)
- DirectoryInfo targetDir = new DirectoryInfo(Path.Combine(destDir.FullName, entry.Name));
- if (entry.IsDirectory)
- {
- targetDir.Create();
- }
- else
- {
- if (targetDir.Parent != null)
- {
- // be on the safe side: do not rely on that directories are always extracted
- // before their children (although this makes sense, but is it guaranteed?)
- targetDir.Parent.Create();
- }
-
- if (fileName != String.Empty)
- {
- using (FileStream streamWriter = File.Create(entry.Name))
- {
- int size = 2048;
- byte[] data = new byte[2048];
- while (true)
- {
- size = s.Read(data, 0, data.Length);
- if (size > 0)
- {
- streamWriter.Write(data, 0, size);
- }
- else
- {
- break;
- }
- }
- }
- }
- }
- }
- }
-
- /*
- ZipFile zipFile = new ZipFile(zipName);
- IEnumerator<FileInfo> entries = zipFile.entries();
-
- while (entries.MoveNext())
- {
- ZipEntry entry = entries.Current;
-
- Stream @in = zipFile.getInputStream(entry);
- DirectoryInfo targetFile = new DirectoryInfo(destDir, entry.Name);
- if (entry.Directory)
- {
- // allow unzipping with directory structure
- //targetFile.mkdirs();
- targetFile.Create();
- }
- else
- {
- if (targetFile.ParentFile != null)
- {
- // be on the safe side: do not rely on that directories are always extracted
- // before their children (although this makes sense, but is it guaranteed?)
- targetFile.ParentFile.mkdirs();
- }
- Stream @out = new BufferedOutputStream(new FileOutputStream(targetFile));
-
- byte[] buffer = new byte[8192];
- int len;
- while ((len = @in.Read(buffer, 0, buffer.Length)) >= 0)
- {
- @out.Write(buffer, 0, len);
- }
-
- @in.Close();
- @out.Close();
- }
- }
-
- zipFile.close();*/
- }
-
public static void SyncConcurrentMerges(IndexWriter writer)
{
SyncConcurrentMerges(writer.Config.MergeScheduler);