diff --git a/src/compression.jl b/src/compression.jl index 40ead54..9ea85f4 100644 --- a/src/compression.jl +++ b/src/compression.jl @@ -4,7 +4,60 @@ abstract type CompressorCodec <: TranscodingStreams.Codec end function Base.show(io::IO, codec::CompressorCodec) - print(io, summary(codec), "(level=$(codec.level), windowbits=$(codec.windowbits))") + print(io, summary(codec)) + print(io, "(level=") + print(io, codec.level) + input_windowbits = mod(abs(codec.windowbits), 16) + if input_windowbits != Z_DEFAULT_WINDOWBITS + print(io, ", windowbits=") + print(io, input_windowbits) + end + if codec.strategy != Z_DEFAULT_STRATEGY + print(io, ", strategy=") + print(io, codec.strategy) + end + print(io, ")") +end + +const level_docs = """ +- `level::Integer=-1` (-1..9): The compression level. + + 1 gives best speed, 9 gives best compression, 0 gives no compression at all + (the input data is simply copied a block at a time). -1 + requests a default compromise between speed and compression (currently + equivalent to level 6). +""" + +const windowbits_docs = """ +- `windowbits::Integer=$(Z_DEFAULT_WINDOWBITS)` (9..15): The size of the history buffer is `2^windowbits`. +""" + +const strategy_docs = """ +- `strategy::Integer=$(Z_DEFAULT_STRATEGY)` ($(Z_DEFAULT_STRATEGY)..$(Z_FIXED)): The compression strategy. + + - $(Z_DEFAULT_STRATEGY) (`Z_DEFAULT_STRATEGY`) is used for normal data. + - $(Z_FILTERED) (`Z_FILTERED`) is used for data produced by a filter (or predictor). + Filtered data consists mostly of small values with a somewhat random + distribution. In this case, the compression algorithm is tuned to compress + them better. The effect of `Z_FILTERED` is to force more Huffman coding + and less string matching; it is somewhat intermediate between + `Z_DEFAULT_STRATEGY` and `Z_HUFFMAN_ONLY`. + - $(Z_HUFFMAN_ONLY) (`Z_HUFFMAN_ONLY`) forces Huffman encoding only (no string match). + - $(Z_RLE) (`Z_RLE`) limits match distances to one (run-length encoding). `Z_RLE` + is designed to be almost as fast as `Z_HUFFMAN_ONLY`, but gives better + compression for PNG image data. + - $(Z_FIXED) (`Z_FIXED`) prevents the use of dynamic Huffman codes, allowing for a + simpler decoder for special applications. +""" + +function check_compressor_args(level, windowbits, strategy) + if !(-1 ≤ level ≤ 9) + throw(ArgumentError("compression level must be within -1..9")) + elseif !(9 ≤ windowbits ≤ 15) + throw(ArgumentError("windowbits must be within 9..15")) + elseif !(0 ≤ strategy ≤ 4) + throw(ArgumentError("strategy must be within 0..4")) + end end @@ -15,31 +68,30 @@ struct GzipCompressor <: CompressorCodec zstream::ZStream level::Int windowbits::Int + strategy::Int end """ - GzipCompressor(;level=$(Z_DEFAULT_COMPRESSION), windowbits=$(Z_DEFAULT_WINDOWBITS)) + GzipCompressor(;level=$(Z_DEFAULT_COMPRESSION), windowbits=$(Z_DEFAULT_WINDOWBITS), strategy=$(Z_DEFAULT_STRATEGY)) Create a gzip compression codec. Arguments --------- -- `level` (-1..9): compression level. 1 gives best speed, 9 gives best compression, 0 gives no compression at all (the input data is simply copied a block at a time). -1 requests a default compromise between speed and compression (currently equivalent to level 6). -- `windowbits` (9..15): size of history buffer is `2^windowbits`. +$(level_docs) +$(windowbits_docs) +$(strategy_docs) !!! warning `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. """ function GzipCompressor(;level::Integer=Z_DEFAULT_COMPRESSION, - windowbits::Integer=Z_DEFAULT_WINDOWBITS) - if !(-1 ≤ level ≤ 9) - throw(ArgumentError("compression level must be within -1..9")) - elseif !(9 ≤ windowbits ≤ 15) - throw(ArgumentError("windowbits must be within 9..15")) - end + windowbits::Integer=Z_DEFAULT_WINDOWBITS, + strategy::Integer=Z_DEFAULT_STRATEGY) + check_compressor_args(level, windowbits, strategy) zstream = ZStream() finalizer(compress_finalizer!, zstream) - return GzipCompressor(zstream, level, windowbits+16) + return GzipCompressor(zstream, level, windowbits+16, strategy) end const GzipCompressorStream{S} = TranscodingStream{GzipCompressor,S} where S<:IO @@ -53,7 +105,7 @@ Create a gzip compression stream (see `GzipCompressor` for `kwargs`). `serialize` and `deepcopy` will not work with this stream due to stored raw pointers. """ function GzipCompressorStream(stream::IO; kwargs...) - x, y = splitkwargs(kwargs, (:level, :windowbits)) + x, y = splitkwargs(kwargs, (:level, :windowbits, :strategy)) return TranscodingStream(GzipCompressor(;x...), stream; y...) end @@ -65,31 +117,30 @@ struct ZlibCompressor <: CompressorCodec zstream::ZStream level::Int windowbits::Int + strategy::Int end """ - ZlibCompressor(;level=$(Z_DEFAULT_COMPRESSION), windowbits=$(Z_DEFAULT_WINDOWBITS)) + ZlibCompressor(;level=$(Z_DEFAULT_COMPRESSION), windowbits=$(Z_DEFAULT_WINDOWBITS), strategy=$(Z_DEFAULT_STRATEGY)) Create a zlib compression codec. Arguments --------- -- `level` (-1..9): compression level. 1 gives best speed, 9 gives best compression, 0 gives no compression at all (the input data is simply copied a block at a time). -1 requests a default compromise between speed and compression (currently equivalent to level 6). -- `windowbits` (9..15): size of history buffer is `2^windowbits`. +$(level_docs) +$(windowbits_docs) +$(strategy_docs) !!! warning `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. """ function ZlibCompressor(;level::Integer=Z_DEFAULT_COMPRESSION, - windowbits::Integer=Z_DEFAULT_WINDOWBITS) - if !(-1 ≤ level ≤ 9) - throw(ArgumentError("compression level must be within -1..9")) - elseif !(9 ≤ windowbits ≤ 15) - throw(ArgumentError("windowbits must be within 9..15")) - end + windowbits::Integer=Z_DEFAULT_WINDOWBITS, + strategy::Integer=Z_DEFAULT_STRATEGY) + check_compressor_args(level, windowbits, strategy) zstream = ZStream() finalizer(compress_finalizer!, zstream) - return ZlibCompressor(zstream, level, windowbits) + return ZlibCompressor(zstream, level, windowbits, strategy) end const ZlibCompressorStream{S} = TranscodingStream{ZlibCompressor,S} where S<:IO @@ -103,7 +154,7 @@ Create a zlib compression stream (see `ZlibCompressor` for `kwargs`). `serialize` and `deepcopy` will not work with this stream due to stored raw pointers. """ function ZlibCompressorStream(stream::IO; kwargs...) - x, y = splitkwargs(kwargs, (:level, :windowbits)) + x, y = splitkwargs(kwargs, (:level, :windowbits, :strategy)) return TranscodingStream(ZlibCompressor(;x...), stream; y...) end @@ -115,31 +166,30 @@ struct DeflateCompressor <: CompressorCodec zstream::ZStream level::Int windowbits::Int + strategy::Int end """ - DeflateCompressor(;level=$(Z_DEFAULT_COMPRESSION), windowbits=$(Z_DEFAULT_WINDOWBITS)) + DeflateCompressor(;level=$(Z_DEFAULT_COMPRESSION), windowbits=$(Z_DEFAULT_WINDOWBITS), strategy=$(Z_DEFAULT_STRATEGY)) Create a deflate compression codec. Arguments --------- -- `level` (-1..9): compression level. 1 gives best speed, 9 gives best compression, 0 gives no compression at all (the input data is simply copied a block at a time). -1 requests a default compromise between speed and compression (currently equivalent to level 6). -- `windowbits` (9..15): size of history buffer is `2^windowbits`. +$(level_docs) +$(windowbits_docs) +$(strategy_docs) !!! warning `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. """ function DeflateCompressor(;level::Integer=Z_DEFAULT_COMPRESSION, - windowbits::Integer=Z_DEFAULT_WINDOWBITS) - if !(-1 ≤ level ≤ 9) - throw(ArgumentError("compression level must be within -1..9")) - elseif !(9 ≤ windowbits ≤ 15) - throw(ArgumentError("windowbits must be within 9..15")) - end + windowbits::Integer=Z_DEFAULT_WINDOWBITS, + strategy::Integer=Z_DEFAULT_STRATEGY) + check_compressor_args(level, windowbits, strategy) zstream = ZStream() finalizer(compress_finalizer!, zstream) - return DeflateCompressor(zstream, level, -Int(windowbits)) + return DeflateCompressor(zstream, level, -Int(windowbits), strategy) end const DeflateCompressorStream{S} = TranscodingStream{DeflateCompressor,S} where S<:IO @@ -153,7 +203,7 @@ Create a deflate compression stream (see `DeflateCompressor` for `kwargs`). `serialize` and `deepcopy` will not work with this stream due to stored raw pointers. """ function DeflateCompressorStream(stream::IO; kwargs...) - x, y = splitkwargs(kwargs, (:level, :windowbits)) + x, y = splitkwargs(kwargs, (:level, :windowbits, :strategy)) return TranscodingStream(DeflateCompressor(;x...), stream; y...) end @@ -163,7 +213,7 @@ end function TranscodingStreams.startproc(codec::CompressorCodec, state::Symbol, error_ref::Error) if codec.zstream.state == C_NULL - code = deflate_init!(codec.zstream, codec.level, codec.windowbits) + code = deflate_init!(codec.zstream, codec.level, codec.windowbits, codec.strategy) # errors in deflate_init! do not require clean up, so just throw if code == Z_OK return :ok diff --git a/src/decompression.jl b/src/decompression.jl index 65a0c9d..939f7e7 100644 --- a/src/decompression.jl +++ b/src/decompression.jl @@ -4,9 +4,22 @@ abstract type DecompressorCodec <: TranscodingStreams.Codec end function Base.show(io::IO, codec::DecompressorCodec) - print(io, summary(codec), "(windowbits=$(codec.windowbits))") + input_windowbits = mod(abs(codec.windowbits), 16) + gziponly = 16 ≤ codec.windowbits ≤ 31 + print(io, summary(codec)) + print(io, "(") + if input_windowbits != Z_DEFAULT_WINDOWBITS + print(io, "windowbits=") + print(io, input_windowbits) + gziponly && print(io, ", ") + end + gziponly && print(io, "gziponly=true") + print(io, ")") end +const decompressor_windowbits_docs = """ +- `windowbits::Integer=$(Z_DEFAULT_WINDOWBITS)` (8..15): Changing `windowbits` from its default of $(Z_DEFAULT_WINDOWBITS) will prevent decoding data using a history buffer larger than `2^windowbits`. +""" # Gzip # ---- @@ -25,8 +38,8 @@ If `gziponly` is `false`, this codec can decompress the zlib format as well. Arguments --------- -- `windowbits` (8..15): Changing `windowbits` from its default of 15 will prevent decoding data using a history buffer larger than `2^windowbits`. -- `gziponly`: flag to inactivate data format detection +$(decompressor_windowbits_docs) +- `gziponly::Bool=false`: If `true`, inactivate data format detection. !!! warning `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. @@ -71,7 +84,7 @@ Create a zlib decompression codec. Arguments --------- -- `windowbits` (8..15): Changing `windowbits` from its default of 15 will prevent decoding data using a history buffer larger than `2^windowbits`. +$(decompressor_windowbits_docs) !!! warning `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. @@ -116,7 +129,7 @@ Create a deflate decompression codec. Arguments --------- -- `windowbits` (8..15): Changing `windowbits` from its default of 15 will prevent decoding data using a history buffer larger than `2^windowbits`. +$(decompressor_windowbits_docs) !!! warning `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. diff --git a/src/libz.jl b/src/libz.jl index ca62fb8..b45bffe 100644 --- a/src/libz.jl +++ b/src/libz.jl @@ -72,7 +72,11 @@ const Z_FINISH = Cint(4) # The deflate compression method const Z_DEFLATED = Cint(8) -const Z_DEFAULT_STRATEGY = Cint(0) +const Z_FILTERED = Cint(1) +const Z_HUFFMAN_ONLY = Cint(2) +const Z_RLE = Cint(3) +const Z_FIXED = Cint(4) +const Z_DEFAULT_STRATEGY = Cint(0) const Z_DEFAULT_MEMLEVEL = Cint(8) const Z_DEFAULT_WINDOWBITS = Cint(15) @@ -85,8 +89,17 @@ end # The `_init!` functions will return an error if the library is not compatible. const zlib_version = "1.3.1" -function deflate_init!(zstream::ZStream, level::Integer, windowbits::Integer) - return ccall((:deflateInit2_, libz), Cint, (Ref{ZStream}, Cint, Cint, Cint, Cint, Cint, Cstring, Cint), zstream, level, Z_DEFLATED, windowbits, #=default memlevel=#8, #=default strategy=#0, zlib_version, sizeof(ZStream)) +function deflate_init!(zstream::ZStream, level::Integer, windowbits::Integer, strategy::Integer=Z_DEFAULT_STRATEGY) + @ccall libz.deflateInit2_( + zstream::Ref{ZStream}, + level::Cint, + Z_DEFLATED::Cint, + windowbits::Cint, + Z_DEFAULT_MEMLEVEL::Cint, + strategy::Cint, + zlib_version::Cstring, + sizeof(ZStream)::Cint, + )::Cint end function deflate_reset!(zstream::ZStream) diff --git a/test/runtests.jl b/test/runtests.jl index 5572537..e8a0aed 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -47,13 +47,11 @@ end @testset "Gzip Codec" begin codec = GzipCompressor() @test codec isa GzipCompressor - @test occursin(r"^(CodecZlib\.)?GzipCompressor\(level=-1, windowbits=\d+\)$", sprint(show, codec)) @test CodecZlib.initialize(codec) === nothing @test CodecZlib.finalize(codec) === nothing codec = GzipDecompressor() @test codec isa GzipDecompressor - @test occursin(r"^(CodecZlib\.)?GzipDecompressor\(windowbits=\d+\)$", sprint(show, codec)) @test CodecZlib.initialize(codec) === nothing @test CodecZlib.finalize(codec) === nothing @@ -149,13 +147,11 @@ end @testset "Zlib Codec" begin codec = ZlibCompressor() @test codec isa ZlibCompressor - @test occursin(r"^(CodecZlib\.)?ZlibCompressor\(level=-1, windowbits=\d+\)$", sprint(show, codec)) @test CodecZlib.initialize(codec) === nothing @test CodecZlib.finalize(codec) === nothing codec = ZlibDecompressor() @test codec isa ZlibDecompressor - @test occursin(r"^(CodecZlib\.)?ZlibDecompressor\(windowbits=\d+\)$", sprint(show, codec)) @test CodecZlib.initialize(codec) === nothing @test CodecZlib.finalize(codec) === nothing @@ -229,13 +225,11 @@ end @testset "Deflate Codec" begin codec = DeflateCompressor() @test codec isa DeflateCompressor - @test occursin(r"^(CodecZlib\.)?DeflateCompressor\(level=-1, windowbits=-\d+\)$", sprint(show, codec)) @test CodecZlib.initialize(codec) === nothing @test CodecZlib.finalize(codec) === nothing codec = DeflateDecompressor() @test codec isa DeflateDecompressor - @test occursin(r"^(CodecZlib\.)?DeflateDecompressor\(windowbits=-\d+\)$", sprint(show, codec)) @test CodecZlib.initialize(codec) === nothing @test CodecZlib.finalize(codec) === nothing @@ -397,3 +391,41 @@ end @test sprint(Base.showerror, ZlibError("test error message")) == "ZlibError: test error message" end +@testset "show $T" for T in (GzipCompressor, ZlibCompressor, DeflateCompressor) + @test repr(T()) == repr(T)*"(level=-1)" + for level in 0:9 + @test repr(T(;level)) == repr(T)*"(level=$(level))" + end + for windowbits in 9:14 + @test repr(T(;windowbits)) == repr(T)*"(level=-1, windowbits=$(windowbits))" + end + for strategy in 1:4 + @test repr(T(;strategy)) == repr(T)*"(level=-1, strategy=$(strategy))" + end + @test repr(T(;level=3, windowbits=13, strategy=2)) == repr(T)*"(level=3, windowbits=13, strategy=2)" +end +@testset "show $T" for T in (GzipDecompressor, ZlibDecompressor, DeflateDecompressor) + @test repr(T()) == repr(T)*"()" + for windowbits in 9:14 + @test repr(T(;windowbits)) == repr(T)*"(windowbits=$(windowbits))" + end + if T == GzipDecompressor + @test repr(T(;gziponly=true)) == repr(T)*"(gziponly=true)" + @test repr(T(;gziponly=true, windowbits=10)) == repr(T)*"(windowbits=10, gziponly=true)" + end +end +@testset "strategy" begin + d = generate_data() + for (encoder, decoder) in [ + (GzipCompressorStream, GzipDecompressorStream), + (ZlibCompressorStream, ZlibDecompressorStream), + (DeflateCompressorStream, DeflateDecompressorStream), + ] + for strategy in 0:4 + c = read(encoder(IOBuffer(d); strategy)) + @test d == read(decoder(IOBuffer(c))) + end + @test_throws ArgumentError encoder(IOBuffer(d); strategy=-1) + @test_throws ArgumentError encoder(IOBuffer(d); strategy=5) + end +end