Repository navigation
Add XZ (LZMA) compression support to System.IO.Compression #1542
Description
Background and motivation
The goal of this proposal is to expose LZMA/XZ compression and decompression through APIs that follow the established patterns in System.IO.Compression.
The native library will be shipped with #124003
API Proposal
namespace System.IO.Compression
{
public enum XzChecksumType
{
None = 0,
Crc32 = 1,
Crc64 = 2,
Sha256 = 3,
}
public sealed partial class XzCompressionOptions
{
public XzCompressionOptions() { }
public static int DefaultQuality { get { throw null; } }
public static int DefaultWindowLog { get { throw null; } }
public static int MaxQuality { get { throw null; } }
public static int MaxWindowLog { get { throw null; } }
public static int MinQuality { get { throw null; } }
public static int MinWindowLog { get { throw null; } }
public System.IO.Compression.XzChecksumType Checksum { get { throw null; } set { } }
public int WindowLog { get { throw null; } set { } }
public bool EnableExtremeMode { get { throw null; } set { } }
public int Quality { get { throw null; } set { } }
}
public sealed partial class XzDecoder : System.IDisposable
{
public XzDecoder() { }
public XzDecoder(int maxWindowLog) { }
public System.Buffers.OperationStatus Decompress(System.ReadOnlySpan<byte> source, System.Span<byte> destination, out int bytesConsumed, out int bytesWritten) { throw null; }
public void Dispose() { }
public static bool TryDecompress(System.ReadOnlySpan<byte> source, System.Span<byte> destination, out int bytesWritten) { throw null; }
}
public sealed partial class XzEncoder : System.IDisposable
{
public XzEncoder() { }
public XzEncoder(int quality) { }
public XzEncoder(int quality, int windowLog) { }
public XzEncoder(System.IO.Compression.XzCompressionOptions compressionOptions) { }
public System.Buffers.OperationStatus Compress(System.ReadOnlySpan<byte> source, System.Span<byte> destination, out int bytesConsumed, out int bytesWritten, bool isFinalBlock) { throw null; }
public void Dispose() { }
public System.Buffers.OperationStatus Flush(System.Span<byte> destination, out int bytesWritten) { throw null; }
public static long GetMaxCompressedLength(long inputLength) { throw null; }
public static bool TryCompress(System.ReadOnlySpan<byte> source, System.Span<byte> destination, out int bytesWritten) { throw null; }
public static bool TryCompress(System.ReadOnlySpan<byte> source, System.Span<byte> destination, out int bytesWritten, int quality) { throw null; }
public static bool TryCompress(System.ReadOnlySpan<byte> source, System.Span<byte> destination, out int bytesWritten, int quality, int windowLog) { throw null; }
}
public sealed partial class XzStream : System.IO.Stream
{
public XzStream(System.IO.Stream stream, System.IO.Compression.CompressionLevel compressionLevel, bool leaveOpen = false) { }
public XzStream(System.IO.Stream stream, System.IO.Compression.CompressionMode mode, bool leaveOpen = false) { }
public XzStream(System.IO.Stream stream, System.IO.Compression.XzCompressionOptions compressionOptions, bool leaveOpen = false) { }
public System.IO.Stream BaseStream { get { throw null; } }
// ... Standard Stream overrides
public override bool CanRead { get { throw null; } }
public override bool CanSeek { get { throw null; } }
public override bool CanWrite { get { throw null; } }
public override long Length { get { throw null; } }
public override long Position { get { throw null; } set { } }
public override System.IAsyncResult BeginRead(byte[] buffer, int offset, int count, System.AsyncCallback? callback, object? state) { throw null; }
public override System.IAsyncResult BeginWrite(byte[] buffer, int offset, int count, System.AsyncCallback? callback, object? state) { throw null; }
protected override void Dispose(bool disposing) { }
public override System.Threading.Tasks.ValueTask DisposeAsync() { throw null; }
public override int EndRead(System.IAsyncResult asyncResult) { throw null; }
public override void EndWrite(System.IAsyncResult asyncResult) { }
public override void Flush() { }
public override System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; }
public override int Read(byte[] buffer, int offset, int count) { throw null; }
public override int Read(System.Span<byte> buffer) { throw null; }
public override System.Threading.Tasks.Task<int> ReadAsync(byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) { throw null; }
public override System.Threading.Tasks.ValueTask<int> ReadAsync(System.Memory<byte> buffer, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
public override int ReadByte() { throw null; }
public override long Seek(long offset, System.IO.SeekOrigin origin) { throw null; }
public override void SetLength(long value) { }
public override void Write(byte[] buffer, int offset, int count) { }
public override void Write(System.ReadOnlySpan<byte> buffer) { }
public override System.Threading.Tasks.Task WriteAsync(byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) { throw null; }
public override System.Threading.Tasks.ValueTask WriteAsync(System.ReadOnlyMemory<byte> buffer, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
public override void WriteByte(byte value) { }
}
} Alternative Designs
-
Use raw byte values for dictionary/window size instead of log₂ value
LZMA natively uses raw byte values for dict_size (e.g., 8388608 for 8 MiB), while Zstandard uses log₂ values forwindowLog. We chose log₂ to maintain consistency withZstandardCompressionOptions.WindowLog. -
Don't expose
EnableExtremeMode
TheEnableExtremeModeflag (maps toLZMA_PRESET_EXTREME)provides marginal compression improvement at significant speed cost, and no other compression API inSystem.IO.Compressionexposes such an analogous toggle.
Activity
GSPP commentedon Jun 24, 2016
LZMA is pretty much the state of the art general purpose compression algorithm. It pushes the Pareto Frontier outwards in quite a large range.
If there is interest in adding more algorithms there would be room for a super fast one that compresses in the range 10-300MB/s. Such algorithms exist and can be found using any search engine.
ericstj commentedon Jun 24, 2016
I've already ported the C# impl for internal use in CLI: https://github.com/dotnet/cli/tree/rel/1.0.0/src/Microsoft.DotNet.Archive/LZMA
The work to do to pull this in is the following:
For perf we could do a similar thing to what was done for ZLIB where we just pick up the native impl and add that to CLRCompression.
@GSPP if you have requests for other algos please file a separate issue.
ianhays commentedon Jun 27, 2016
Something we should consider before we start working on adding support for Tar/Deflate/7z/LZMA/LZMA2/LZ4/BZ2/etc is how we want to design the APIs for the classes to keep them relatively similar without cluttering up Sys.IO.Compression. Options there off the top of my head:
I imagine we'll want to go with the first option so we can provide fine-grained control of the algorithm variables, but its worth considering nonetheless.
ericstj commentedon Jun 27, 2016
First option is what I was thinking. I think we can look at higher-level simple to use archive APIs that might allow for selection of the compression algorithm for an archive where the caller doesn't necessarily need to know about the specific algo type or params but we'd also want to support fine-grained composition of these things to allow for maximum flexibility/tuning. Also for reading APIs should be able to "sniff" a data stream to determine what it is (if the format allows for that deterministically) without forcing the user to tell us via type instantiation.
ianhays commentedon Jun 27, 2016
I think this would be the most happy compromise between full functionality and ease of use. I'm picturing a bunch of algorithm-specific streams (DeflateStream/GZipStream/LZMAStream) as well as one high-level stream (CompressionStream) that has simple write/read functionality using the defaults of the chosen compression type e.g.
That would be ideal. The algorithm-specific APIs (e.g. DeflateStream) should require the given stream adhere to the format, but the higher-level API should attempt to detect the header type and default (probably to Deflate) it it can't be determined.
ericstj commentedon Jun 27, 2016
Seems reasonable so long as it doesn't hurt perf (eg: cause an additional buffer copy). I'd say that's a separate feature, but definitely one worth looking at.
ianhays commentedon Jun 27, 2016
Agreed. I opened https://github.com/dotnet/corefx/issues/9709 to track the discussion on that.
GSPP commentedon Jun 28, 2016
I think the enum approach suffers from non-extensibility. Wouldn't it be better to use one derived type for each algorithm? Algorithms have different parameters and sometimes different capabilities. For example, Zip has the capability to add named entries (files). Many compression libraries offer a ZipStream that behaves like a normal stream plus the ability to define files.
Proposal: Make
CompressionStreaman abstract base class that anyone including user code can derive from. That way there is a standardized infrastructure for the compression mode and leaving the stream open. Also, maybe we can have standardized compression levels 1-9 that each derived algorithm interprets and uses to set it's settings.Also, I do not see in what way a class-based approach (as opposed to an enum-based approach) would be inferior(?). Seems equal or better in every regard (apart from format detection).
I advise against format detection when decompressing. The format should be statically known and usually is. Some formats are headerless which breaks the scheme.
ericstj commentedon Jun 28, 2016
Good point.
I don't see how
CompressionStreamreally solves this. We already have theStreamabstraction and all the things we're talking about controlling are on construction which isn't part of the abstraction. The only things we might expose on the stream are getters for construction parameters but I don't know if that provides enough value to justify another type in the heirarchy.I don't think we disagree. @ianhays was talking about some convenience construction pattern that would use the public
Streamderived types behind the scenes. One option for the construction pattern would be enums, but as you mention these don't version well. Perhaps we have a typeCompressionParametersthat has levels like you suggest and compression Stream implementations can translate those to appropriate algo-specific params.Let's use https://github.com/dotnet/corefx/issues/9709 to track coming up with a convenience construction pattern and use this issue for describing the new streams we intend to add.
ianhays commentedon Jul 6, 2016
Back to the LZMA discussion:
Not sure how we're going to want to do this since the LZMA SDK includes implementations in C as well as C#. Some options and related thoughts:
GSPP commentedon Jul 7, 2016
Ideally, it would be a native code solution where possible with a managed fallback. How much slower is the C# version? My guess would be 2-3x. C compilers excel at this kind of code.
ianhays commentedon Jul 8, 2016
Alright, I've got native and managed versions working in Sys.IO.Compression on Windows so we can get some early perf results. Properties are set to the defaults for both.
As with ZLib, the lzma native implementation is significantly faster.Wall-clock time for 10 compress/decompress of the files in the Canterbury Corpus:Note that the native lzma here is doing File IO itself, so it is going to be a little bit faster than what the end-product would be since we'd want to do the IO in C# so LZMA could follow the Stream pattern.
GSPP commentedon Jul 8, 2016
I wonder why native and managed have so different results.
kennedy.xlsis smaller with managed.kennedy.xlshas a crazy managed compression time. Weird.78 remaining items
bartonjs commentedon Apr 14, 2026
Video
Looks good as proposed
rzikm commentedon Jun 10, 2026
After long and thorough discussion, we decided not to move forward with this API in .NET 11. There are several reasons, including:
We will keep the issue open for the time being. We are open to feedback and discussion about our points above. If we missed anything important, please let us know.
KieranDevvs commentedon Jun 10, 2026
Rather disappointing considering there has been a TON of community feedback for this. I don't really understand the argument of "we already have this algorithm that's different but does the same thing" when many of us have LZMA files that we don't control but need to consume and are having to rely on custom wrappers around other native implementations and some of us are just out right blocked from doing anything with LZMA because of this.
karelz commentedon Jun 17, 2026
@KieranDevvs we understand your disappointment.
However, it is not clear to us, why do you think that our implementation would be any better than your custom wrappers around native implementation. We planned to pick one community native implementation and embed it anyway. What is preventing you from doing exactly that, if you need support specifically for LZMA?
Our main concern is that LZMA does not seem to be the way forward, therefore making everyone pay the penalty of larger size of embedded native implementation + the associated ongoing maintenance cost do not sound like a good investment for .NET.
Are we missing something here?
jjxtra commentedon Jun 17, 2026
What about an "official" nuget package that isn't part of the core .NET offering?
karelz commentedon Jun 17, 2026
@jjxtra that solves size angle, but has same (if not a bit larger) maintenance cost.
For something that is being replaced by Zstd, we are not sure it is well spend time/effort/energy for us. We believe we can add more value elsewhere in the repo, where there are no alternatives / workarounds for community.
KieranDevvs commentedon Jun 17, 2026
Its not that wrapping a native implementation is in any way bad, its that you're asking everyone to write their own (which isnt a trivial task) for something which I consider a basic (from the consumer perspective, not the implementation perspective) IO function. Now maybe that's not necessarily bad either but you have to look at the community support for this and ask yourself why the landscape is so barren even though there are millions if not billions of LZMA archives out there and the standard has existed for almost 30 years and there are tons of people trying to all do the same thing, yet failing. From my experience its because the documentation is extremely lacking and the API surfaces usually aren't great to consume, and/or there's a performance issue.
As @JIXTRA suggested, the size concern can be mitigated via an isolated NuGet package, and while I understand your concern about having to support a library that isnt the latest and greatest thing, it doesn't sit right with me for a few reasons:
Yes LZMA may or may not be the way forward, but that is irrelevant IMO. The metrics that I think you should be concerned with are the community need for this (which has been proven to be desirable and high on many fronts (this GitHub issue, StackOverflow questions, and many failed community NuGet packages). Like I have previously mentioned, many people end up in a situation needing to read and deflate an LZMA archive or stream and they don't own the data and they cant change the archival format.
Supporting LZMA isnt like supporting a dying or dead technology, LZMA is still very much in use all over the planet and while something like zstandard can offer similar ratios, that doesn't change the fact that LZMA is still widely used and will be for years if not decades to come, and I don't see any evidence of that number declining just because an alternative exists.
This might be my ignorance speaking so forgive me if I'm wrong, but how much support are you expecting to really be giving to this if it were to be added to the .NET ecosystem? The LZMA algorithm is extremely stable, and the 7zip implementation seems to get security updates every few months or so and there are on average between 1 and 3 on demand CVE's to patch which will be done by the native implementation provider (I'm assuming this will be 7zips).
As I previously touched on, there is no easy way to consume these files in .NET. This proposal brings a clear OOB unified way on doing this.
And finally, why have we spent an entire decade requesting this to be added, gone through API design and approval and then at the last minute canned it all because someone suddenly doesn't like the idea of supporting it. Considering the .NET team keep asking people to give feedback and engagement around the things they would like to see prioritised, this seems like a huge slap in the face and extremely discouraging to engage with anything else.
I understand that there are pros and cons to this, but in my opinion the pros outweigh the cons quite significantly.
karelz commentedon Jun 17, 2026
I think this speaks actually in the opposite - the fact we cared enough to go with the API proposal, implementation, etc. (albeit after long time) demonstrates we DO CARE about community feedback and we take it seriously. We use it for prioritization.
There was not a whim of one person "killing it". Some (valid) concerns were raised in last phases of reviews about sustainability of our approach and we realized the whole effort is rather non-productive for .NET team to do based on the forward looking approach.
We believe that maintenance of the package is a high burden which we will regret later on. Obviously, we are very careful about that.
We are still open to change our mind, if there is very strong signal from the community, but the above is our current thinking.
Technically speaking, it does not have RFC or ISO standard. The best we could find is a draft inside LZMA SDK. From practical purpose, it might be sufficient. But it is yet another small difference compared to Zstd.
If anyone is interested, we would be happy to contribute our API design and implementation we have to any OSS project. That way perhaps most of the asks above can be satisfied?
Would that be viable middle ground?
NiKiZe commentedon Oct 15, 2026
Regardless of Zstandard not being comparable other than better in speed (not compression) there is still the fact, as metioned that there is requirements to both consume and generate lzma and xz data. Instead of using one standard implementation, now everyone needs to grab their own. So I, as many others, fail to see how "we have zstandard instead" is an argument here, that to me shows lack of understanding on why the feature is actually requested, but following that logic I assume that any other compression support is removed in net11 as well? (why shouldn't it, now that there is zstandard)