diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # lzlib +## 0.1.1.0 + + * Add a higher-level API + * Add `Show` and `Eq` instances to `LZErrno` type + * Bundle with `lzlib` version 1.11 + ## 0.1.0.0 Initial release diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # lzlib -Low-level bindings to [lzlib](https://www.nongnu.org/lzip/lzlib.html). +Haskell bindings to [lzlib](https://www.nongnu.org/lzip/lzlib.html). diff --git a/cbits/lzlib.c b/cbits/lzlib.c new file mode 100644 --- /dev/null +++ b/cbits/lzlib.c @@ -0,0 +1,601 @@ +/* Lzlib - Compression library for the lzip format + Copyright (C) 2009-2019 Antonio Diaz Diaz. + + This library is free software. Redistribution and use in source and + binary forms, with or without modification, are permitted provided + that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ + +#include +#include +#include +#include + +#include "lzlib.h" +#include "lzip.h" +#include "cbuffer.c" +#include "decoder.h" +#include "decoder.c" +#include "encoder_base.h" +#include "encoder_base.c" +#include "encoder.h" +#include "encoder.c" +#include "fast_encoder.h" +#include "fast_encoder.c" + + +struct LZ_Encoder + { + unsigned long long partial_in_size; + unsigned long long partial_out_size; + struct LZ_encoder_base * lz_encoder_base; /* these 3 pointers make a */ + struct LZ_encoder * lz_encoder; /* polymorphic encoder */ + struct FLZ_encoder * flz_encoder; + enum LZ_Errno lz_errno; + bool fatal; + }; + +static void LZ_Encoder_init( struct LZ_Encoder * const e ) + { + e->partial_in_size = 0; + e->partial_out_size = 0; + e->lz_encoder_base = 0; + e->lz_encoder = 0; + e->flz_encoder = 0; + e->lz_errno = LZ_ok; + e->fatal = false; + } + + +struct LZ_Decoder + { + unsigned long long partial_in_size; + unsigned long long partial_out_size; + struct Range_decoder * rdec; + struct LZ_decoder * lz_decoder; + enum LZ_Errno lz_errno; + Lzip_header member_header; /* header of current member */ + bool fatal; + bool first_header; /* true until first header is read */ + bool seeking; + }; + +static void LZ_Decoder_init( struct LZ_Decoder * const d ) + { + int i; + d->partial_in_size = 0; + d->partial_out_size = 0; + d->rdec = 0; + d->lz_decoder = 0; + d->lz_errno = LZ_ok; + for( i = 0; i < Lh_size; ++i ) d->member_header[i] = 0; + d->fatal = false; + d->first_header = true; + d->seeking = false; + } + + +static bool verify_encoder( struct LZ_Encoder * const e ) + { + if( !e ) return false; + if( !e->lz_encoder_base || ( !e->lz_encoder && !e->flz_encoder ) || + ( e->lz_encoder && e->flz_encoder ) ) + { e->lz_errno = LZ_bad_argument; return false; } + return true; + } + + +static bool verify_decoder( struct LZ_Decoder * const d ) + { + if( !d ) return false; + if( !d->rdec ) + { d->lz_errno = LZ_bad_argument; return false; } + return true; + } + + +/*------------------------- Misc Functions -------------------------*/ + +const char * LZ_version( void ) { return LZ_version_string; } + + +const char * LZ_strerror( const enum LZ_Errno lz_errno ) + { + switch( lz_errno ) + { + case LZ_ok : return "ok"; + case LZ_bad_argument : return "Bad argument"; + case LZ_mem_error : return "Not enough memory"; + case LZ_sequence_error: return "Sequence error"; + case LZ_header_error : return "Header error"; + case LZ_unexpected_eof: return "Unexpected EOF"; + case LZ_data_error : return "Data error"; + case LZ_library_error : return "Library error"; + } + return "Invalid error code"; + } + + +int LZ_min_dictionary_bits( void ) { return min_dictionary_bits; } +int LZ_min_dictionary_size( void ) { return min_dictionary_size; } +int LZ_max_dictionary_bits( void ) { return max_dictionary_bits; } +int LZ_max_dictionary_size( void ) { return max_dictionary_size; } +int LZ_min_match_len_limit( void ) { return min_match_len_limit; } +int LZ_max_match_len_limit( void ) { return max_match_len; } + + +/*---------------------- Compression Functions ----------------------*/ + +struct LZ_Encoder * LZ_compress_open( const int dictionary_size, + const int match_len_limit, + const unsigned long long member_size ) + { + Lzip_header header; + struct LZ_Encoder * const e = + (struct LZ_Encoder *)malloc( sizeof (struct LZ_Encoder) ); + if( !e ) return 0; + LZ_Encoder_init( e ); + if( !Lh_set_dictionary_size( header, dictionary_size ) || + match_len_limit < min_match_len_limit || + match_len_limit > max_match_len || + member_size < min_dictionary_size ) + e->lz_errno = LZ_bad_argument; + else + { + if( dictionary_size == 65535 && match_len_limit == 16 ) + { + e->flz_encoder = (struct FLZ_encoder *)malloc( sizeof (struct FLZ_encoder) ); + if( e->flz_encoder && FLZe_init( e->flz_encoder, member_size ) ) + { e->lz_encoder_base = &e->flz_encoder->eb; return e; } + free( e->flz_encoder ); e->flz_encoder = 0; + } + else + { + e->lz_encoder = (struct LZ_encoder *)malloc( sizeof (struct LZ_encoder) ); + if( e->lz_encoder && LZe_init( e->lz_encoder, Lh_get_dictionary_size( header ), + match_len_limit, member_size ) ) + { e->lz_encoder_base = &e->lz_encoder->eb; return e; } + free( e->lz_encoder ); e->lz_encoder = 0; + } + e->lz_errno = LZ_mem_error; + } + e->fatal = true; + return e; + } + + +int LZ_compress_close( struct LZ_Encoder * const e ) + { + if( !e ) return -1; + if( e->lz_encoder_base ) + { LZeb_free( e->lz_encoder_base ); + free( e->lz_encoder ); free( e->flz_encoder ); } + free( e ); + return 0; + } + + +int LZ_compress_finish( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) || e->fatal ) return -1; + Mb_finish( &e->lz_encoder_base->mb ); + /* if (open --> write --> finish) use same dictionary size as lzip. */ + /* this does not save any memory. */ + if( Mb_data_position( &e->lz_encoder_base->mb ) == 0 && + Re_member_position( &e->lz_encoder_base->renc ) == Lh_size ) + { + Mb_adjust_dictionary_size( &e->lz_encoder_base->mb ); + Lh_set_dictionary_size( e->lz_encoder_base->renc.header, + e->lz_encoder_base->mb.dictionary_size ); + e->lz_encoder_base->renc.cb.buffer[5] = e->lz_encoder_base->renc.header[5]; + } + return 0; + } + + +int LZ_compress_restart_member( struct LZ_Encoder * const e, + const unsigned long long member_size ) + { + if( !verify_encoder( e ) || e->fatal ) return -1; + if( !LZeb_member_finished( e->lz_encoder_base ) ) + { e->lz_errno = LZ_sequence_error; return -1; } + if( member_size < min_dictionary_size ) + { e->lz_errno = LZ_bad_argument; return -1; } + + e->partial_in_size += Mb_data_position( &e->lz_encoder_base->mb ); + e->partial_out_size += Re_member_position( &e->lz_encoder_base->renc ); + + if( e->lz_encoder ) LZe_reset( e->lz_encoder, member_size ); + else FLZe_reset( e->flz_encoder, member_size ); + e->lz_errno = LZ_ok; + return 0; + } + + +int LZ_compress_sync_flush( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) || e->fatal ) return -1; + if( !Mb_flushing_or_end( &e->lz_encoder_base->mb ) ) + e->lz_encoder_base->mb.flushing = true; + return 0; + } + + +int LZ_compress_read( struct LZ_Encoder * const e, + uint8_t * const buffer, const int size ) + { + int out_size = 0; + if( !verify_encoder( e ) || e->fatal ) return -1; + if( size < 0 ) return 0; + do { + if( ( e->flz_encoder && !FLZe_encode_member( e->flz_encoder ) ) || + ( e->lz_encoder && !LZe_encode_member( e->lz_encoder ) ) ) + { e->lz_errno = LZ_library_error; e->fatal = true; return -1; } + if( e->lz_encoder_base->mb.flushing && + Mb_available_bytes( &e->lz_encoder_base->mb ) <= 0 && + LZeb_sync_flush( e->lz_encoder_base ) ) + e->lz_encoder_base->mb.flushing = false; + out_size += Re_read_data( &e->lz_encoder_base->renc, + buffer + out_size, size - out_size ); + } + while( e->lz_encoder_base->mb.flushing && out_size < size && + Mb_enough_available_bytes( &e->lz_encoder_base->mb ) && + Re_enough_free_bytes( &e->lz_encoder_base->renc ) ); + return out_size; + } + + +int LZ_compress_write( struct LZ_Encoder * const e, + const uint8_t * const buffer, const int size ) + { + if( !verify_encoder( e ) || e->fatal ) return -1; + return Mb_write_data( &e->lz_encoder_base->mb, buffer, size ); + } + + +int LZ_compress_write_size( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) || e->fatal ) return -1; + return Mb_free_bytes( &e->lz_encoder_base->mb ); + } + + +enum LZ_Errno LZ_compress_errno( struct LZ_Encoder * const e ) + { + if( !e ) return LZ_bad_argument; + return e->lz_errno; + } + + +int LZ_compress_finished( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return -1; + return ( Mb_data_finished( &e->lz_encoder_base->mb ) && + LZeb_member_finished( e->lz_encoder_base ) ); + } + + +int LZ_compress_member_finished( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return -1; + return LZeb_member_finished( e->lz_encoder_base ); + } + + +unsigned long long LZ_compress_data_position( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return 0; + return Mb_data_position( &e->lz_encoder_base->mb ); + } + + +unsigned long long LZ_compress_member_position( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return 0; + return Re_member_position( &e->lz_encoder_base->renc ); + } + + +unsigned long long LZ_compress_total_in_size( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return 0; + return e->partial_in_size + Mb_data_position( &e->lz_encoder_base->mb ); + } + + +unsigned long long LZ_compress_total_out_size( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return 0; + return e->partial_out_size + Re_member_position( &e->lz_encoder_base->renc ); + } + + +/*--------------------- Decompression Functions ---------------------*/ + +struct LZ_Decoder * LZ_decompress_open( void ) + { + struct LZ_Decoder * const d = + (struct LZ_Decoder *)malloc( sizeof (struct LZ_Decoder) ); + if( !d ) return 0; + LZ_Decoder_init( d ); + + d->rdec = (struct Range_decoder *)malloc( sizeof (struct Range_decoder) ); + if( !d->rdec || !Rd_init( d->rdec ) ) + { + if( d->rdec ) { Rd_free( d->rdec ); free( d->rdec ); d->rdec = 0; } + d->lz_errno = LZ_mem_error; d->fatal = true; + } + return d; + } + + +int LZ_decompress_close( struct LZ_Decoder * const d ) + { + if( !d ) return -1; + if( d->lz_decoder ) + { LZd_free( d->lz_decoder ); free( d->lz_decoder ); } + if( d->rdec ) { Rd_free( d->rdec ); free( d->rdec ); } + free( d ); + return 0; + } + + +int LZ_decompress_finish( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) || d->fatal ) return -1; + if( d->seeking ) + { d->seeking = false; d->partial_in_size += Rd_purge( d->rdec ); } + else Rd_finish( d->rdec ); + return 0; + } + + +int LZ_decompress_reset( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + if( d->lz_decoder ) + { LZd_free( d->lz_decoder ); free( d->lz_decoder ); d->lz_decoder = 0; } + d->partial_in_size = 0; + d->partial_out_size = 0; + Rd_reset( d->rdec ); + d->lz_errno = LZ_ok; + d->fatal = false; + d->first_header = true; + d->seeking = false; + return 0; + } + + +int LZ_decompress_sync_to_member( struct LZ_Decoder * const d ) + { + unsigned skipped = 0; + if( !verify_decoder( d ) ) return -1; + if( d->lz_decoder ) + { LZd_free( d->lz_decoder ); free( d->lz_decoder ); d->lz_decoder = 0; } + if( Rd_find_header( d->rdec, &skipped ) ) d->seeking = false; + else + { + if( !d->rdec->at_stream_end ) d->seeking = true; + else { d->seeking = false; d->partial_in_size += Rd_purge( d->rdec ); } + } + d->partial_in_size += skipped; + d->lz_errno = LZ_ok; + d->fatal = false; + return 0; + } + + +int LZ_decompress_read( struct LZ_Decoder * const d, + uint8_t * const buffer, const int size ) + { + int result; + if( !verify_decoder( d ) ) return -1; + if( d->fatal ) /* don't return error until pending bytes are read */ + { if( d->lz_decoder && !Cb_empty( &d->lz_decoder->cb ) ) goto get_data; + return -1; } + if( d->seeking || size < 0 ) return 0; + + if( d->lz_decoder && LZd_member_finished( d->lz_decoder ) ) + { + d->partial_out_size += LZd_data_position( d->lz_decoder ); + LZd_free( d->lz_decoder ); free( d->lz_decoder ); d->lz_decoder = 0; + } + if( !d->lz_decoder ) + { + int rd; + d->partial_in_size += d->rdec->member_position; + d->rdec->member_position = 0; + if( Rd_available_bytes( d->rdec ) < Lh_size + 5 && + !d->rdec->at_stream_end ) return 0; + if( Rd_finished( d->rdec ) && !d->first_header ) return 0; + rd = Rd_read_data( d->rdec, d->member_header, Lh_size ); + if( rd < Lh_size || Rd_finished( d->rdec ) ) /* End Of File */ + { + if( rd <= 0 || Lh_verify_prefix( d->member_header, rd ) ) + d->lz_errno = LZ_unexpected_eof; + else + d->lz_errno = LZ_header_error; + d->fatal = true; + return -1; + } + if( !Lh_verify_magic( d->member_header ) ) + { + /* unreading the header prevents sync_to_member from skipping a member + if leading garbage is shorter than a full header; "lgLZIP\x01\x0C" */ + if( Rd_unread_data( d->rdec, rd ) ) + { + if( d->first_header || !Lh_verify_corrupt( d->member_header ) ) + d->lz_errno = LZ_header_error; + else + d->lz_errno = LZ_data_error; /* corrupt header */ + } + else + d->lz_errno = LZ_library_error; + d->fatal = true; + return -1; + } + if( !Lh_verify_version( d->member_header ) || + !isvalid_ds( Lh_get_dictionary_size( d->member_header ) ) ) + { + /* Skip a possible "LZIP" leading garbage; "LZIPLZIP\x01\x0C". + Leave member_pos pointing to the first error. */ + if( Rd_unread_data( d->rdec, 1 + !Lh_verify_version( d->member_header ) ) ) + d->lz_errno = LZ_data_error; /* bad version or bad dict size */ + else + d->lz_errno = LZ_library_error; + d->fatal = true; + return -1; + } + d->first_header = false; + if( Rd_available_bytes( d->rdec ) < 5 ) + { + /* set position at EOF */ + d->rdec->member_position += Cb_used_bytes( &d->rdec->cb ); + Cb_reset( &d->rdec->cb ); + d->lz_errno = LZ_unexpected_eof; + d->fatal = true; + return -1; + } + d->lz_decoder = (struct LZ_decoder *)malloc( sizeof (struct LZ_decoder) ); + if( !d->lz_decoder || !LZd_init( d->lz_decoder, d->rdec, + Lh_get_dictionary_size( d->member_header ) ) ) + { /* not enough free memory */ + if( d->lz_decoder ) + { LZd_free( d->lz_decoder ); free( d->lz_decoder ); d->lz_decoder = 0; } + d->lz_errno = LZ_mem_error; + d->fatal = true; + return -1; + } + d->rdec->reload_pending = true; + } + result = LZd_decode_member( d->lz_decoder ); + if( result != 0 ) + { + if( result == 2 ) /* set input position at EOF */ + { d->rdec->member_position += Cb_used_bytes( &d->rdec->cb ); + Cb_reset( &d->rdec->cb ); + d->lz_errno = LZ_unexpected_eof; } + else if( result == 5 ) d->lz_errno = LZ_library_error; + else d->lz_errno = LZ_data_error; + d->fatal = true; + if( Cb_empty( &d->lz_decoder->cb ) ) return -1; + } +get_data: + return Cb_read_data( &d->lz_decoder->cb, buffer, size ); + } + + +int LZ_decompress_write( struct LZ_Decoder * const d, + const uint8_t * const buffer, const int size ) + { + int result; + if( !verify_decoder( d ) || d->fatal ) return -1; + if( size < 0 ) return 0; + + result = Rd_write_data( d->rdec, buffer, size ); + while( d->seeking ) + { + int size2; + unsigned skipped = 0; + if( Rd_find_header( d->rdec, &skipped ) ) d->seeking = false; + d->partial_in_size += skipped; + if( result >= size ) break; + size2 = Rd_write_data( d->rdec, buffer + result, size - result ); + if( size2 > 0 ) result += size2; + else break; + } + return result; + } + + +int LZ_decompress_write_size( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) || d->fatal ) return -1; + return Rd_free_bytes( d->rdec ); + } + + +enum LZ_Errno LZ_decompress_errno( struct LZ_Decoder * const d ) + { + if( !d ) return LZ_bad_argument; + return d->lz_errno; + } + + +int LZ_decompress_finished( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) || d->fatal ) return -1; + return ( Rd_finished( d->rdec ) && + ( !d->lz_decoder || LZd_member_finished( d->lz_decoder ) ) ); + } + + +int LZ_decompress_member_finished( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) || d->fatal ) return -1; + return ( d->lz_decoder && LZd_member_finished( d->lz_decoder ) ); + } + + +int LZ_decompress_member_version( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + return Lh_version( d->member_header ); + } + + +int LZ_decompress_dictionary_size( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + return Lh_get_dictionary_size( d->member_header ); + } + + +unsigned LZ_decompress_data_crc( struct LZ_Decoder * const d ) + { + if( verify_decoder( d ) && d->lz_decoder ) + return LZd_crc( d->lz_decoder ); + return 0; + } + + +unsigned long long LZ_decompress_data_position( struct LZ_Decoder * const d ) + { + if( verify_decoder( d ) && d->lz_decoder ) + return LZd_data_position( d->lz_decoder ); + return 0; + } + + +unsigned long long LZ_decompress_member_position( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return 0; + return d->rdec->member_position; + } + + +unsigned long long LZ_decompress_total_in_size( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return 0; + return d->partial_in_size + d->rdec->member_position; + } + + +unsigned long long LZ_decompress_total_out_size( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return 0; + if( d->lz_decoder ) + return d->partial_out_size + LZd_data_position( d->lz_decoder ); + return d->partial_out_size; + } diff --git a/cbits/lzlib.h b/cbits/lzlib.h new file mode 100644 --- /dev/null +++ b/cbits/lzlib.h @@ -0,0 +1,106 @@ +/* Lzlib - Compression library for the lzip format + Copyright (C) 2009-2019 Antonio Diaz Diaz. + + This library is free software. Redistribution and use in source and + binary forms, with or without modification, are permitted provided + that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +#define LZ_API_VERSION 1 + +static const char * const LZ_version_string = "1.11"; + +enum LZ_Errno { LZ_ok = 0, LZ_bad_argument, LZ_mem_error, + LZ_sequence_error, LZ_header_error, LZ_unexpected_eof, + LZ_data_error, LZ_library_error }; + + +const char * LZ_version( void ); +const char * LZ_strerror( const enum LZ_Errno lz_errno ); + +int LZ_min_dictionary_bits( void ); +int LZ_min_dictionary_size( void ); +int LZ_max_dictionary_bits( void ); +int LZ_max_dictionary_size( void ); +int LZ_min_match_len_limit( void ); +int LZ_max_match_len_limit( void ); + + +/*---------------------- Compression Functions ----------------------*/ + +struct LZ_Encoder; + +struct LZ_Encoder * LZ_compress_open( const int dictionary_size, + const int match_len_limit, + const unsigned long long member_size ); +int LZ_compress_close( struct LZ_Encoder * const encoder ); + +int LZ_compress_finish( struct LZ_Encoder * const encoder ); +int LZ_compress_restart_member( struct LZ_Encoder * const encoder, + const unsigned long long member_size ); +int LZ_compress_sync_flush( struct LZ_Encoder * const encoder ); + +int LZ_compress_read( struct LZ_Encoder * const encoder, + uint8_t * const buffer, const int size ); +int LZ_compress_write( struct LZ_Encoder * const encoder, + const uint8_t * const buffer, const int size ); +int LZ_compress_write_size( struct LZ_Encoder * const encoder ); + +enum LZ_Errno LZ_compress_errno( struct LZ_Encoder * const encoder ); +int LZ_compress_finished( struct LZ_Encoder * const encoder ); +int LZ_compress_member_finished( struct LZ_Encoder * const encoder ); + +unsigned long long LZ_compress_data_position( struct LZ_Encoder * const encoder ); +unsigned long long LZ_compress_member_position( struct LZ_Encoder * const encoder ); +unsigned long long LZ_compress_total_in_size( struct LZ_Encoder * const encoder ); +unsigned long long LZ_compress_total_out_size( struct LZ_Encoder * const encoder ); + + +/*--------------------- Decompression Functions ---------------------*/ + +struct LZ_Decoder; + +struct LZ_Decoder * LZ_decompress_open( void ); +int LZ_decompress_close( struct LZ_Decoder * const decoder ); + +int LZ_decompress_finish( struct LZ_Decoder * const decoder ); +int LZ_decompress_reset( struct LZ_Decoder * const decoder ); +int LZ_decompress_sync_to_member( struct LZ_Decoder * const decoder ); + +int LZ_decompress_read( struct LZ_Decoder * const decoder, + uint8_t * const buffer, const int size ); +int LZ_decompress_write( struct LZ_Decoder * const decoder, + const uint8_t * const buffer, const int size ); +int LZ_decompress_write_size( struct LZ_Decoder * const decoder ); + +enum LZ_Errno LZ_decompress_errno( struct LZ_Decoder * const decoder ); +int LZ_decompress_finished( struct LZ_Decoder * const decoder ); +int LZ_decompress_member_finished( struct LZ_Decoder * const decoder ); + +int LZ_decompress_member_version( struct LZ_Decoder * const decoder ); +int LZ_decompress_dictionary_size( struct LZ_Decoder * const decoder ); +unsigned LZ_decompress_data_crc( struct LZ_Decoder * const decoder ); + +unsigned long long LZ_decompress_data_position( struct LZ_Decoder * const decoder ); +unsigned long long LZ_decompress_member_position( struct LZ_Decoder * const decoder ); +unsigned long long LZ_decompress_total_in_size( struct LZ_Decoder * const decoder ); +unsigned long long LZ_decompress_total_out_size( struct LZ_Decoder * const decoder ); + +#ifdef __cplusplus +} +#endif diff --git a/lzlib.cabal b/lzlib.cabal --- a/lzlib.cabal +++ b/lzlib.cabal @@ -1,6 +1,6 @@ cabal-version: 1.18 name: lzlib -version: 0.1.0.0 +version: 0.1.1.0 license: BSD3 license-file: LICENSE copyright: Copyright: (c) 2019 Vanessa McHale @@ -9,6 +9,7 @@ synopsis: lzlib bindings description: Lzlib bindings via [c2hs](http://hackage.haskell.org/package/c2hs). + Includes a bundled copy of lzlib category: Codec, Compression build-type: Simple @@ -27,15 +28,46 @@ manual: True library - exposed-modules: Codec.Lzip + exposed-modules: + Codec.Lzip + Codec.Lzip.Raw + + c-sources: cbits/lzlib.c hs-source-dirs: src default-language: Haskell2010 - extra-libraries: lz + other-extensions: BangPatterns + include-dirs: cbits + install-includes: cbits/lzlib.h ghc-options: -Wall - build-depends: base >=4.3 && <5 + build-depends: + base >=4.9 && <5, + bytestring -any if !flag(cross) build-tool-depends: c2hs:c2hs -any + + if impl(ghc >=8.0) + ghc-options: + -Wincomplete-uni-patterns -Wincomplete-record-updates + -Wredundant-constraints + + if impl(ghc >=8.4) + ghc-options: -Wmissing-export-lists + +test-suite lzlib-test + type: exitcode-stdio-1.0 + main-is: Spec.hs + hs-source-dirs: test + default-language: Haskell2010 + ghc-options: -threaded -rtsopts -with-rtsopts=-N -Wall + build-depends: + base -any, + lzlib -any, + hspec -any, + bytestring -any, + filepattern -any, + filepath -any, + directory -any if impl(ghc >=8.0) ghc-options: diff --git a/src/Codec/Lzip.chs b/src/Codec/Lzip.chs deleted file mode 100644 --- a/src/Codec/Lzip.chs +++ /dev/null @@ -1,114 +0,0 @@ -module Codec.Lzip ( -- * Prolegomena - LZErrno (..) - , lZVersion - , lZStrerror - , lZMinDictionaryBits - , lZMinDictionarySize - , lZMaxDictionaryBits - , lZMaxDictionarySize - , lZMinMatchLenLimit - , lZMaxMatchLenLimit - , UInt8 - -- * Compression functions - , LZEncoder - , LZEncoderPtr - , lZCompressOpen - , lZCompressClose - , lZCompressFinish - , lZCompressRestartMember - , lZCompressSyncFlush - , lZCompressRead - , lZCompressWrite - , lZCompressWriteSize - , lZCompressErrno - , lZCompressFinished - , lZCompressMemberFinished - , lZCompressDataPosition - , lZCompressMemberPosition - , lZCompressTotalInSize - , lZCompressTotalOutSize - -- * Decompression functions - , LZDecoder - , LZDecoderPtr - , lZDecompressOpen - , lZDecompressClose - , lZDecompressFinish - , lZDecompressReset - , lZDecompressSyncToMember - , lZDecompressRead - , lZDecompressWrite - , lZDecompressWriteSize - , lZDecompressErrno - , lZDecompressFinished - , lZDecompressMemberFinished - , lZDecompressDictionarySize - , lZDecompressDataCrc - , lZDecompressDataPosition - , lZDecompressMemberPosition - , lZDecompressTotalInSize - , lZDecompressTotalOutSize - ) where - -import Foreign.C.String -import Foreign.C.Types -import Foreign.Ptr (Ptr) - -#include -#include - -type UInt8 = {# type uint8_t #} - -{# enum LZ_Errno as LZErrno {underscoreToCase} #} - -{# fun LZ_version as ^ {} -> `CString' #} -{# fun LZ_strerror as ^ { `LZErrno' } -> `CString' #} -{# fun LZ_min_dictionary_bits as ^ {} -> `CInt' #} -{# fun LZ_min_dictionary_size as ^ {} -> `CInt' #} -{# fun LZ_max_dictionary_bits as ^ {} -> `CInt' #} -{# fun LZ_max_dictionary_size as ^ {} -> `CInt' #} -{# fun LZ_min_match_len_limit as ^ {} -> `CInt' #} -{# fun LZ_max_match_len_limit as ^ {} -> `CInt' #} - --- | Abstract data type -data LZEncoder - -{# pointer *LZ_Encoder as LZEncoderPtr -> LZEncoder #} - -{# fun LZ_compress_open as ^ { `CInt', `CInt', id `CULLong' } -> `LZEncoderPtr' #} -{# fun LZ_compress_close as ^ { `LZEncoderPtr' } -> `CInt' #} -{# fun LZ_compress_finish as ^ { `LZEncoderPtr' } -> `CInt' #} -{# fun LZ_compress_restart_member as ^ { `LZEncoderPtr', id `CULLong' } -> `CInt' #} -{# fun LZ_compress_sync_flush as ^ { `LZEncoderPtr' } -> `CInt' #} -{# fun LZ_compress_read as ^ { `LZEncoderPtr', id `Ptr UInt8', `CInt' } -> `CInt' #} -{# fun LZ_compress_write as ^ { `LZEncoderPtr', id `Ptr UInt8', `CInt' } -> `CInt' #} -{# fun LZ_compress_write_size as ^ { `LZEncoderPtr' } -> `CInt' #} -{# fun LZ_compress_errno as ^ { `LZEncoderPtr' } -> `LZErrno' #} -{# fun LZ_compress_finished as ^ { `LZEncoderPtr' } -> `CInt' #} -{# fun LZ_compress_member_finished as ^ { `LZEncoderPtr' } -> `CInt' #} -{# fun LZ_compress_data_position as ^ { `LZEncoderPtr' } -> `CULLong' id #} -{# fun LZ_compress_member_position as ^ { `LZEncoderPtr' } -> `CULLong' id #} -{# fun LZ_compress_total_in_size as ^ { `LZEncoderPtr' } -> `CULLong' id #} -{# fun LZ_compress_total_out_size as ^ { `LZEncoderPtr' } -> `CULLong' id #} - --- | Abstract data type -data LZDecoder - -{# pointer *LZ_Decoder as LZDecoderPtr -> LZDecoder #} - -{# fun LZ_decompress_open as ^ {} -> `LZDecoderPtr' #} -{# fun LZ_decompress_close as ^ { `LZDecoderPtr' } -> `CInt' #} -{# fun LZ_decompress_finish as ^ { `LZDecoderPtr' } -> `CInt' #} -{# fun LZ_decompress_reset as ^ { `LZDecoderPtr' } -> `CInt' #} -{# fun LZ_decompress_sync_to_member as ^ { `LZDecoderPtr' } -> `CInt' #} -{# fun LZ_decompress_read as ^ { `LZDecoderPtr', id `Ptr UInt8', `CInt' } -> `CInt' #} -{# fun LZ_decompress_write as ^ { `LZDecoderPtr', id `Ptr UInt8', `CInt' } -> `CInt' #} -{# fun LZ_decompress_write_size as ^ { `LZDecoderPtr' } -> `CInt' #} -{# fun LZ_decompress_errno as ^ { `LZDecoderPtr' } -> `LZErrno' #} -{# fun LZ_decompress_finished as ^ { `LZDecoderPtr' } -> `CInt' #} -{# fun LZ_decompress_member_finished as ^ { `LZDecoderPtr' } -> `CInt' #} -{# fun LZ_decompress_dictionary_size as ^ { `LZDecoderPtr' } -> `CInt' #} -{# fun LZ_decompress_data_crc as ^ { `LZDecoderPtr' } -> `CUInt' #} -{# fun LZ_decompress_data_position as ^ { `LZDecoderPtr' } -> `CULLong' id #} -{# fun LZ_decompress_member_position as ^ { `LZDecoderPtr' } -> `CULLong' id #} -{# fun LZ_decompress_total_in_size as ^ { `LZDecoderPtr' } -> `CULLong' id #} -{# fun LZ_decompress_total_out_size as ^ { `LZDecoderPtr' } -> `CULLong' id #} diff --git a/src/Codec/Lzip.hs b/src/Codec/Lzip.hs new file mode 100644 --- /dev/null +++ b/src/Codec/Lzip.hs @@ -0,0 +1,120 @@ +{-# LANGUAGE BangPatterns #-} + +module Codec.Lzip ( compressStrict + , compressWithStrict + , decompressStrict + , CompressionLevel (..) + -- * Low-level bindings + , module Codec.Lzip.Raw + ) where + +import Codec.Lzip.Raw +import Control.Monad (unless, void, when) +import Data.Bits (shiftL) +import qualified Data.ByteString as BS +import Data.Functor (($>)) +import Data.Int (Int64) +import Data.Maybe (fromMaybe) +import Data.Semigroup +import Foreign.Marshal.Alloc (free, mallocBytes) +import Foreign.Ptr (Ptr, castPtr, plusPtr) +import System.IO.Unsafe (unsafePerformIO) + +data CompressionLevel = Zero + | One + | Two + | Three + | Four + | Five + | Six + | Seven + | Eight + | Nine + +data LzOptions = LzOptions { _dictionarySize :: !Int + , _matchLenLimit :: !Int + } + +encoderOptions :: CompressionLevel -> LzOptions +encoderOptions Zero = LzOptions 65535 16 +encoderOptions One = LzOptions (1 `shiftL` 20) 5 +encoderOptions Two = LzOptions (3 `shiftL` 19) 6 +encoderOptions Three = LzOptions (1 `shiftL` 21) 8 +encoderOptions Four = LzOptions (3 `shiftL` 20) 12 +encoderOptions Five = LzOptions (1 `shiftL` 22) 20 +encoderOptions Six = LzOptions (1 `shiftL` 23) 36 +encoderOptions Seven = LzOptions (1 `shiftL` 24) 68 +encoderOptions Eight = LzOptions (3 `shiftL` 23) 132 +encoderOptions Nine = LzOptions (1 `shiftL` 25) 273 + +-- | This does not do any error recovery; for that you should use +-- [lziprecover](https://www.nongnu.org/lzip/lziprecover.html). +{-# NOINLINE decompressStrict #-} +decompressStrict :: BS.ByteString -> BS.ByteString +decompressStrict bs = unsafePerformIO $ BS.useAsCStringLen bs $ \(bytes, sz) -> do + + decoder <- lZDecompressOpen + maxSz <- lZDecompressWriteSize decoder + + let bufMax = min (fromIntegral maxSz) sz + buf <- mallocBytes sz + res <- loop decoder (buf, bufMax) (castPtr bytes, sz) 0 sz mempty + + void $ lZDecompressClose decoder + free buf + + pure res + + where + loop :: LZDecoderPtr -> (Ptr UInt8, Int) -> (Ptr UInt8, Int) -> Int -> Int -> BS.ByteString -> IO BS.ByteString + loop decoder (buf, bufSz) (bytes, sz) offset total !acc = do + let toWrite = min bufSz (total - offset) + bytesWritten <- if offset < total + then Just <$> lZDecompressWrite decoder (bytes `plusPtr` offset) (fromIntegral toWrite) + else Nothing <$ lZDecompressFinish decoder + res <- lZDecompressFinished decoder + if res == 1 + then pure acc + else do + let newOffset = offset + fromIntegral (fromMaybe 0 bytesWritten) + bytesRead <- lZDecompressRead decoder buf (fromIntegral bufSz) + bsActual <- BS.packCStringLen (castPtr buf, fromIntegral bytesRead) + loop decoder (buf, bufSz) (bytes, sz) newOffset total (acc <> bsActual) + +{-# NOINLINE compressStrict #-} +compressStrict :: BS.ByteString -> BS.ByteString +compressStrict = compressWithStrict Nine + +-- TODO: memory use +{-# NOINLINE compressWithStrict #-} +compressWithStrict :: CompressionLevel -> BS.ByteString -> BS.ByteString +compressWithStrict level bs = unsafePerformIO $ BS.useAsCStringLen bs $ \(bytes, sz) -> do + + encoder <- lZCompressOpen (fromIntegral dictionarySize) (fromIntegral matchLenLimit) (fromIntegral memberSize) + + void $ lZCompressWrite encoder (castPtr bytes) (fromIntegral sz) + void $ lZCompressFinish encoder + + newBytes <- mallocBytes sz + res <- readLoop encoder (newBytes, sz) 0 mempty + + void $ lZCompressClose encoder + + pure res + + where + + readLoop :: LZEncoderPtr -> (Ptr UInt8, Int) -> Int -> BS.ByteString -> IO BS.ByteString + readLoop encoder (buf, sz) bytesRead acc = do + bytesActual <- lZCompressRead encoder buf (fromIntegral sz) + res <- lZCompressFinished encoder + bsActual <- BS.packCStringLen (castPtr buf, fromIntegral bytesActual) + if res == 1 + then pure (acc <> bsActual) + else readLoop encoder (buf, sz) (bytesRead + fromIntegral bytesActual) (acc <> bsActual) + + memberSize :: Int64 + memberSize = maxBound + + dictionarySize = _dictionarySize $ encoderOptions level + matchLenLimit = _matchLenLimit $ encoderOptions level diff --git a/src/Codec/Lzip/Raw.chs b/src/Codec/Lzip/Raw.chs new file mode 100644 --- /dev/null +++ b/src/Codec/Lzip/Raw.chs @@ -0,0 +1,120 @@ +-- | Consult the lzlib [documentation](https://www.nongnu.org/lzip/manual/lzlib_manual.html) +-- for more details +module Codec.Lzip.Raw ( -- * Prolegomena + LZErrno (..) + , lZVersion + , lZStrerror + , lZMinDictionaryBits + , lZMinDictionarySize + , lZMaxDictionaryBits + , lZMaxDictionarySize + , lZMinMatchLenLimit + , lZMaxMatchLenLimit + , UInt8 + -- * Compression functions + , LZEncoder + , LZEncoderPtr + , lZCompressOpen + , lZCompressClose + , lZCompressFinish + , lZCompressRestartMember + , lZCompressSyncFlush + , lZCompressRead + , lZCompressWrite + , lZCompressWriteSize + , lZCompressErrno + , lZCompressFinished + , lZCompressMemberFinished + , lZCompressDataPosition + , lZCompressMemberPosition + , lZCompressTotalInSize + , lZCompressTotalOutSize + -- * Decompression functions + , LZDecoder + , LZDecoderPtr + , lZDecompressOpen + , lZDecompressClose + , lZDecompressFinish + , lZDecompressReset + , lZDecompressSyncToMember + , lZDecompressRead + , lZDecompressWrite + , lZDecompressWriteSize + , lZDecompressErrno + , lZDecompressFinished + , lZDecompressMemberFinished + , lZDecompressDictionarySize + , lZDecompressDataCrc + , lZDecompressDataPosition + , lZDecompressMemberPosition + , lZDecompressTotalInSize + , lZDecompressTotalOutSize + ) where + +import Foreign.C.Types +import Foreign.Ptr (Ptr) + +#include +#include + +type UInt8 = {# type uint8_t #} +{#typedef uint8_t UInt8#} +{#default in `Ptr UInt8' [uint8_t *] id#} + +{# enum LZ_Errno as LZErrno {underscoreToCase} deriving (Eq) #} + +{# fun pure LZ_version as ^ {} -> `String' #} +{# fun pure LZ_strerror as ^ { `LZErrno' } -> `String' #} +{# fun LZ_min_dictionary_bits as ^ {} -> `CInt' #} +{# fun LZ_min_dictionary_size as ^ {} -> `CInt' #} +{# fun LZ_max_dictionary_bits as ^ {} -> `CInt' #} +{# fun LZ_max_dictionary_size as ^ {} -> `CInt' #} +{# fun LZ_min_match_len_limit as ^ {} -> `CInt' #} +{# fun LZ_max_match_len_limit as ^ {} -> `CInt' #} + +instance Show LZErrno where + show = lZStrerror + +-- | Abstract data type +data LZEncoder + +{# pointer *LZ_Encoder as LZEncoderPtr -> LZEncoder #} + +{# fun LZ_compress_open as ^ { `CInt', `CInt', id `CULLong' } -> `LZEncoderPtr' #} +{# fun LZ_compress_close as ^ { `LZEncoderPtr' } -> `CInt' #} +{# fun LZ_compress_finish as ^ { `LZEncoderPtr' } -> `CInt' #} +{# fun LZ_compress_restart_member as ^ { `LZEncoderPtr', id `CULLong' } -> `CInt' #} +{# fun LZ_compress_sync_flush as ^ { `LZEncoderPtr' } -> `CInt' #} +{# fun LZ_compress_read as ^ { `LZEncoderPtr', `Ptr UInt8', `CInt' } -> `CInt' #} +{# fun LZ_compress_write as ^ { `LZEncoderPtr', `Ptr UInt8', `CInt' } -> `CInt' #} +{# fun LZ_compress_write_size as ^ { `LZEncoderPtr' } -> `CInt' #} +{# fun LZ_compress_errno as ^ { `LZEncoderPtr' } -> `LZErrno' #} +{# fun LZ_compress_finished as ^ { `LZEncoderPtr' } -> `CInt' #} +{# fun LZ_compress_member_finished as ^ { `LZEncoderPtr' } -> `CInt' #} +{# fun LZ_compress_data_position as ^ { `LZEncoderPtr' } -> `CULLong' id #} +{# fun LZ_compress_member_position as ^ { `LZEncoderPtr' } -> `CULLong' id #} +{# fun LZ_compress_total_in_size as ^ { `LZEncoderPtr' } -> `CULLong' id #} +{# fun LZ_compress_total_out_size as ^ { `LZEncoderPtr' } -> `CULLong' id #} + +-- | Abstract data type +data LZDecoder + +{# pointer *LZ_Decoder as LZDecoderPtr -> LZDecoder #} + +{# fun LZ_decompress_open as ^ {} -> `LZDecoderPtr' #} +{# fun LZ_decompress_close as ^ { `LZDecoderPtr' } -> `CInt' #} +{# fun LZ_decompress_finish as ^ { `LZDecoderPtr' } -> `CInt' #} +{# fun LZ_decompress_reset as ^ { `LZDecoderPtr' } -> `CInt' #} +{# fun LZ_decompress_sync_to_member as ^ { `LZDecoderPtr' } -> `CInt' #} +{# fun LZ_decompress_read as ^ { `LZDecoderPtr', `Ptr UInt8', `CInt' } -> `CInt' #} +{# fun LZ_decompress_write as ^ { `LZDecoderPtr', `Ptr UInt8', `CInt' } -> `CInt' #} +{# fun LZ_decompress_write_size as ^ { `LZDecoderPtr' } -> `CInt' #} +{# fun LZ_decompress_errno as ^ { `LZDecoderPtr' } -> `LZErrno' #} +{# fun LZ_decompress_finished as ^ { `LZDecoderPtr' } -> `CInt' #} +{# fun LZ_decompress_member_finished as ^ { `LZDecoderPtr' } -> `CInt' #} +{# fun LZ_decompress_dictionary_size as ^ { `LZDecoderPtr' } -> `CInt' #} +{# fun LZ_decompress_data_crc as ^ { `LZDecoderPtr' } -> `CUInt' #} +{# fun LZ_decompress_data_position as ^ { `LZDecoderPtr' } -> `CULLong' id #} +{# fun LZ_decompress_member_position as ^ { `LZDecoderPtr' } -> `CULLong' id #} +{# fun LZ_decompress_total_in_size as ^ { `LZDecoderPtr' } -> `CULLong' id #} +{# fun LZ_decompress_total_out_size as ^ { `LZDecoderPtr' } -> `CULLong' id #} diff --git a/stack.yaml b/stack.yaml --- a/stack.yaml +++ b/stack.yaml @@ -1,4 +1,4 @@ --- -resolver: lts-14.4 +resolver: lts-14.5 packages: - '.' diff --git a/test/Spec.hs b/test/Spec.hs new file mode 100644 --- /dev/null +++ b/test/Spec.hs @@ -0,0 +1,25 @@ +module Main ( main ) where + +import Codec.Lzip +import qualified Data.ByteString as BS +import Data.Foldable (traverse_) +import System.Directory (doesDirectoryExist) +import System.FilePath (()) +import System.FilePattern.Directory (getDirectoryFiles) +import Test.Hspec + +compressFile :: FilePath -> Spec +compressFile fp = parallel $ + it "roundtrip should be identity" $ do + str <- BS.readFile fp + decompressStrict (compressStrict str) `shouldBe` str + +main :: IO () +main = do + b <- doesDirectoryExist "dist-newstyle" + libs <- if b + then getDirectoryFiles "dist-newstyle" ["**/*.so", "**/*.dll"] + else pure [] + hspec $ do + describe "decompress/compress" $ + traverse_ compressFile (("dist-newstyle" ) <$> libs)