0.3.0-alpha

ChangeLog.md

@@ -0,0 +1,3 @@
+### 0.3.0-alpha
+
+Initial publish

README.md

@@ -1,6 +1,155 @@
 # FastLZMA2Net
- Fast LZMA2 Wrapper for .NET
 
-# ⚠️Library under construction⚠️
+Fast LZMA2 Wrapper for .NET
+
+⚠️Library is alpha test⚠️
+
+⚠️API may have breaking change⚠️
+
+[Change Log](ChangeLog.md)
+
+With respect to [Fast LZMA2](https://github.com/conor42/fast-lzma2)
+
+the most powerful compression algorithm ever.
+
+![](./readme/benchmark.png)
+
+# Requirements
+
+**Windows x86/64 .NET 8 runtime**
+
+It is reconmended using x64. 
+
+x86 arch may have potential malfunction.
+
+# Installation
+
+`PM> Install-Package FastLZMA2Net -prerelease`
+
+# Usage
+
+## Simple compression
+
+When using compression occasionally
+
+``` c#
+string SourceFilePath = @"D:\dummy.tar";
+string CompressedFilePath = @"D:\dummy.tar.fl2";
+string DecompressedFilePath = @"D:\dummy.recovery.tar";
+
+// Simple compression
+byte[] origin = File.ReadAllBytes(SourceFilePath);
+byte[] compressed = FL2.Compress(origin,0);
+byte[] decompressed = FL2.Decompress(compressed);
+```
+
+## Context Compression
+
+When you have many small file, consider using context.
+
+```c#
+// Context compression, context can be reuse.
+Compressor compressor = new(0) { CompressLevel = 10 };
+compressed = compressor.Compress(origin);
+compressed = compressor.Compress(origin);
+compressed = compressor.Compress(origin);
+
+Decompressor decompressor = new Decompressor();
+decompressed = decompressor.Decompress(compressed);
+decompressed = decompressor.Decompress(compressed);
+decompressed = decompressor.Decompress(compressed);
+```
+
+
+## Streaming Compression 
+
+When you have a very large file (>2GB) or slow I/O
+
+``` c# 
+byte[] buffer = new byte[256 * 1024 * 1024]; 
+// use 256MB input buffer 
+// This is suitable for most cases
+``` 
+### small file or data (<2GB)
+``` c# 
+// compress
+using (MemoryStream ms = new MemoryStream())
+{
+    using (CompressionStream cs = new CompressionStream(ms))
+    {
+        cs.Write(origin);
+    }
+    compressed = ms.ToArray();
+}
+// decompress
+using (MemoryStream recoveryStream = new MemoryStream())
+{
+    using (MemoryStream ms = new MemoryStream(compressed))
+    {
+        using (DecompressionStream ds = new DecompressionStream(ms))
+        {
+            ds.CopyTo(recoveryStream);
+        }
+    }
+    decompress = recoveryStream.ToArray();
+}
+```
+
+### Large file or data (>2GB)
+
+When processing Large file, It is not acceptable reading all data into memory.
+
+And .NET array have size limit <2GB 
+
+It is recommended to using DFA(direct file access) streaming.
+
+have tested on a 7GB file. 
+
+Streaming Might have potential malfunction.
+```c#
+//large file streaming compression using Direct file access(>2GB)
+using (FileStream compressedFile = File.OpenWrite(CompressedFilePath))
+{
+    using (CompressionStream cs = new CompressionStream(compressedFile))
+    {
+        using (FileStream sourceFile = File.OpenRead(SourceFilePath))
+        {
+            //DO NOT USE sourceFile.CopyTo(cs)
+            // CopyTo() calls Write() inside, which terminate stream after 1 cycle.
+            long offset = 0;
+            while (offset < sourceFile.Length)
+            {
+                long remaining = sourceFile.Length - offset;
+                int bytesToWrite = (int)Math.Min(64 * 1024 * 1024, remaining);
+                sourceFile.Read(buffer, 0, bytesToWrite);
+                cs.Append(buffer, 0, bytesToWrite);
+                offset += bytesToWrite;
+            }
+            // make sure always use Flush() after all Append() complete
+            // Flush() add checksum to stream and finish streaming.
+            cs.Flush();
+        }
+    }
+}
+
+//large file streaming decompress(>2GB)
+using (FileStream recoveryStream = File.OpenWrite(DecompressedFilePath))
+{
+    using (FileStream compressedFile = File.OpenRead(CompressedFilePath))
+    {
+        using (DecompressionStream ds = new DecompressionStream(compressedFile))
+        {
+            ds.CopyTo(recoveryStream);
+        }
+    }
+}
+```
+
+# Bug report 
+
+Feel free to open an issue.
+
+# Contribution
+
+PR is welcome.
 
-Comming soon

src/Demo/Program.cs

@@ -1,77 +1,83 @@
-using System;
-using System.Diagnostics;
-using System.IO.Compression;
-using System.IO.MemoryMappedFiles;
-using System.Runtime.InteropServices;
-using System.Security.Cryptography;
+using System.Diagnostics;
 using FastLZMA2Net;
-using static System.Runtime.InteropServices.JavaScript.JSType;
 
 namespace Demo
 {
     internal class Program
     {
         private static async Task Main(string[] args)
         {
-            int size = IntPtr.Size;
-            NativeMethods.SetWinDllDirectory();
-            byte[] src = new byte[128*1048576];
-            for (int i = 0; i < src.Length; i++)
-            {
-                src[i] = (byte)(i % 0xf); // 使用模运算确保值在 0x0 到 0xf 之间
-            }
-
-            Compressor compressor = new(0) { CompressLevel = 10,HighCompressLevel=1};
-            Console.WriteLine( compressor.DictSizeProperty);
-            FileInfo srcFile = new FileInfo(@"d:\ffmpeg.exe");
-            FileInfo dstFile = new FileInfo(@"d:\半条命1三合一.fl2");
-            //compressor.Compress(@"d:\半条命1三合一.tar", @"d:\半条命1三合一.fl2");
-            byte[] origin = File.ReadAllBytes(@"D:\半条命1三合一.tar");
-            byte[] compressedRef = File.ReadAllBytes(@"D:\半条命1三合一.fl2");
-
-            using FileStream sourceFile = File.OpenRead(@"D:\Devotion v1.05.tar");
-            using FileStream compressedFile = File.OpenRead(@"D:\Devotion v1.05.tar.fl2");
-            using FileStream recoveryFile = File.OpenWrite(@"D:\recovery.tar");
+            string SourceFilePath = @"D:\dummy.tar";
+            string CompressedFilePath = @"D:\dummy.tar.fl2";
+            string DecompressedFilePath = @"D:\dummy.recovery.tar";
 
-            var length = recoveryFile.Length;
-            byte[] buffer = new byte[256 * 1024 * 1024];
-            //using (DecompressionStream ds = new DecompressionStream(compressedFile))
-            //{
-            //    ds.CopyTo(recoveryFile);
-            //}
+            // Simple compression
+            byte[] origin = File.ReadAllBytes(SourceFilePath);
+            byte[] compressed = FL2.Compress(origin,0);
+            byte[] decompressed = FL2.Decompress(compressed);
 
-            return;
-            Stopwatch sw = new Stopwatch();
-            sw.Start();
-            using (CompressionStream cs = new CompressionStream(compressedFile, outBufferSize: 64 * 1024 * 1024, nbThreads: 0))
+            // Context compression, context can be reuse.
+            Compressor compressor = new(0) { CompressLevel = 10 };
+            compressed = compressor.Compress(origin);
+            compressed = compressor.Compress(origin);
+            compressed = compressor.Compress(origin);
+
+            Decompressor decompressor = new Decompressor();
+            decompressed = decompressor.Decompress(compressed);
+            decompressed = decompressor.Decompress(compressed);
+            decompressed = decompressor.Decompress(compressed);
+
+            // Streaming Compression 
+            byte[] buffer = new byte[256 * 1024 * 1024]; 
+            // use 256MB input buffer
+
+            // small file or data (<2GB)
+            using (MemoryStream ms = new MemoryStream())
             {
-                var clevel = cs.GetParameter(CompressParameterEnum.FL2_p_compressionLevel);
-                long offset = 0;
-                while (offset < sourceFile.Length)
+                using (CompressionStream cs = new CompressionStream(ms))
                 {
-                    long remaining = sourceFile.Length - offset;
-                    int bytesToWrite =(int) Math.Min(64 * 1024 * 1024, remaining);
-                    sourceFile.Read(buffer, 0, bytesToWrite);
-                    cs.Append(buffer,0, bytesToWrite);
-                    offset += bytesToWrite;
+                    cs.Write(origin);
                 }
-                cs.Flush();
+                compressed = ms.ToArray();
             }
 
-            sw.Stop();
-
-            sourceFile.Close();
-            compressedFile.Close();
-            Console.WriteLine($"{sw.Elapsed.TotalSeconds}s");
-        }
+            //large file streaming compression using Direct file access(>2GB)
+            using (FileStream compressedFile = File.OpenWrite(CompressedFilePath))
+            {
+                using (CompressionStream cs = new CompressionStream(compressedFile))
+                {
+                    using (FileStream sourceFile = File.OpenRead(SourceFilePath))
+                    {
+                        //DO NOT USE sourceFile.CopyTo(cs)
+                        // CopyTo calls write() internal, which terminate stream after 1 cycle.
+                        long offset = 0;
+                        while (offset < sourceFile.Length)
+                        {
+                            long remaining = sourceFile.Length - offset;
+                            int bytesToWrite = (int)Math.Min(64 * 1024 * 1024, remaining);
+                            sourceFile.Read(buffer, 0, bytesToWrite);
+                            cs.Append(buffer, 0, bytesToWrite);
+                            offset += bytesToWrite;
+                        }
+                        // make sure always use Flush() after Append()
+                        //Flush() add checksum to stream and finish streaming.
+                        cs.Flush();
+                    }
+                }
+            }
 
-        private static void PrintHex(byte[] byteArray)
-        {
-            foreach (byte b in byteArray)
+            //large file streaming decompress(>2GB)
+            using (FileStream recoveryStream = File.OpenWrite(DecompressedFilePath))
             {
-                Console.Write($"{b:X2} ");
+                using (FileStream compressedFile = File.OpenRead(CompressedFilePath))
+                {
+                    using (DecompressionStream ds = new DecompressionStream(compressedFile))
+                    {
+                        ds.CopyTo(recoveryStream);
+                    }
+                }
             }
-            Console.WriteLine(Environment.NewLine);
+
         }
     }
 }

src/FastLZMA2Net.Test/CStreamTest.cs

@@ -1,12 +1,5 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using System.Diagnostics;
 using FastLZMA2Net;
-using static System.Runtime.InteropServices.JavaScript.JSType;
 
 namespace Test
 {
@@ -61,6 +54,7 @@ public void TestOnetime()
                 Debug.Assert(origin.SequenceEqual(recovery));
             }
         }
+
         [TestMethod]
         public async Task TestAsync()
         {
@@ -78,5 +72,33 @@ public async Task TestAsync()
                 Debug.Assert(origin.SequenceEqual(recovery));
             }
         }
+
+        [TestMethod]
+        public void TestDFA()
+        {
+            byte[] buffer = new byte[1024];
+            using FileStream sourceFile = File.OpenRead(@"Resources/dummy.raw");
+            using FileStream compressedFile = File.OpenWrite(@"Resources/dummy.fl2");
+            using (CompressionStream cs = new CompressionStream(compressedFile, outBufferSize: 1024, nbThreads: 0))
+            {
+                long offset = 0;
+                while (offset < sourceFile.Length)
+                {
+                    long remaining = sourceFile.Length - offset;
+                    int bytesToWrite = (int)Math.Min(1024, remaining);
+                    sourceFile.Read(buffer, 0, bytesToWrite);
+                    cs.Append(buffer, 0, bytesToWrite);
+                    offset += bytesToWrite;
+                }
+                cs.Flush();
+            }
+
+            sourceFile.Close();
+            compressedFile.Close();
+            byte[] compressed = File.ReadAllBytes(@"Resources/dummy.fl2");
+            byte[] recovery = FL2.Decompress(compressed);
+            byte[] origin = File.ReadAllBytes(@"Resources/dummy.raw");
+            Assert.IsTrue(origin.SequenceEqual(recovery));
+        }
     }
-}
+}