lzlib 1.0.7.1 → 1.0.7.2
raw patch · 17 files changed
+224/−210 lines, 17 filesPVP ok
version bump matches the API change (PVP)
API changes (from Hackage documentation)
Files
- CHANGELOG.md +4/−0
- bench/Bench.hs +2/−2
- cbits/cbuffer.c +5/−5
- cbits/decoder.c +7/−14
- cbits/decoder.h +85/−38
- cbits/encoder.c +56/−71
- cbits/encoder.h +2/−3
- cbits/encoder_base.c +10/−13
- cbits/encoder_base.h +8/−14
- cbits/fast_encoder.c +14/−21
- cbits/fast_encoder.h +1/−1
- cbits/lzip.h +2/−1
- cbits/lzlib.c +1/−1
- cbits/lzlib.h +3/−3
- lzlib.cabal +19/−19
- mem/Mem.hs +2/−3
- test/Spec.hs +3/−1
CHANGELOG.md view
@@ -1,5 +1,9 @@ # lzlib +## 1.0.7.2++ * Bump C sources to lzlib 1.13+ ## 1.0.7.1 * Bump C sources to lzlib 1.12
bench/Bench.hs view
@@ -9,8 +9,8 @@ unpack :: FilePath -> IO () unpack fp' = withSystemTempDirectory "lzlib" $- \fp -> BSL.writeFile (fp </> "dump.tar") =<<- (decompress <$> BSL.readFile fp')+ \fp -> BSL.writeFile (fp </> "dump.tar")+ . decompress =<< BSL.readFile fp' pack :: FilePath -> IO () pack fp' = withSystemTempDirectory "lzlib" $
cbits/cbuffer.c view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided@@ -76,9 +76,9 @@ } -/* Copies up to 'out_size' bytes to 'out_buffer' and updates 'get'.+/* Copy up to 'out_size' bytes to 'out_buffer' and update 'get'. If 'out_buffer' is null, the bytes are discarded.- Returns the number of bytes copied or discarded.+ Return the number of bytes copied or discarded. */ static unsigned Cb_read_data( struct Circular_buffer * const cb, uint8_t * const out_buffer,@@ -110,8 +110,8 @@ } -/* Copies up to 'in_size' bytes from 'in_buffer' and updates 'put'.- Returns the number of bytes copied.+/* Copy up to 'in_size' bytes from 'in_buffer' and update 'put'.+ Return the number of bytes copied. */ static unsigned Cb_write_data( struct Circular_buffer * const cb, const uint8_t * const in_buffer,
cbits/decoder.c view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided@@ -49,8 +49,6 @@ while( !Rd_finished( rdec ) ) {- int len;- const int pos_state = LZd_data_position( d ) & pos_state_mask; /* const unsigned mpos = rdec->member_position; if( mpos - old_mpos > rd_min_available_bytes ) return 5; old_mpos = mpos; */@@ -58,23 +56,19 @@ { if( !rdec->at_stream_end ) return 0; if( Cb_empty( &rdec->cb ) ) break; } /* decode until EOF */ if( !LZd_enough_free_bytes( d ) ) return 0;+ const int pos_state = LZd_data_position( d ) & pos_state_mask; if( Rd_decode_bit( rdec, &d->bm_match[*state][pos_state] ) == 0 ) /* 1st bit */ { /* literal byte */ Bit_model * const bm = d->bm_literal[get_lit_state(LZd_peek_prev( d ))];- if( St_is_char( *state ) )- {- *state -= ( *state < 4 ) ? *state : 3;+ if( ( *state = St_set_char( *state ) ) < 4 ) LZd_put_byte( d, Rd_decode_tree8( rdec, bm ) );- } else- {- *state -= ( *state < 10 ) ? 3 : 6; LZd_put_byte( d, Rd_decode_matched( rdec, bm, LZd_peek( d, d->rep0 ) ) );- } continue; } /* match or repeated match */+ int len; if( Rd_decode_bit( rdec, &d->bm_rep[*state] ) != 0 ) /* 2nd bit */ { if( Rd_decode_bit( rdec, &d->bm_rep0[*state] ) == 0 ) /* 3rd bit */@@ -100,13 +94,12 @@ d->rep0 = distance; } *state = St_set_rep( *state );- len = min_match_len + Rd_decode_len( rdec, &d->rep_len_model, pos_state );+ len = Rd_decode_len( rdec, &d->rep_len_model, pos_state ); } else /* match */ {- unsigned distance;- len = min_match_len + Rd_decode_len( rdec, &d->match_len_model, pos_state );- distance = Rd_decode_tree6( rdec, d->bm_dis_slot[get_len_state(len)] );+ len = Rd_decode_len( rdec, &d->match_len_model, pos_state );+ unsigned distance = Rd_decode_tree6( rdec, d->bm_dis_slot[get_len_state(len)] ); if( distance >= start_dis_model ) { const unsigned dis_slot = distance;
cbits/decoder.h view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided@@ -72,8 +72,8 @@ rdec->member_position = 0; rdec->at_stream_end = false; } -/* Seeks a member header and updates 'get'. '*skippedp' is set to the- number of bytes skipped. Returns true if it finds a valid header.+/* Seek for a member header and update 'get'. Set '*skippedp' to the number+ of bytes skipped. Return true if a valid header is found. */ static bool Rd_find_header( struct Range_decoder * const rdec, unsigned * const skippedp )@@ -140,8 +140,7 @@ int i; rdec->reload_pending = false; rdec->code = 0;- for( i = 0; i < 5; ++i )- rdec->code = (rdec->code << 8) | Rd_get_byte( rdec );+ for( i = 0; i < 5; ++i ) rdec->code = (rdec->code << 8) | Rd_get_byte( rdec ); rdec->range = 0xFFFFFFFFU; rdec->code &= rdec->range; /* make sure that first byte is discarded */ }@@ -161,12 +160,11 @@ int i; for( i = num_bits; i > 0; --i ) {- bool bit; Rd_normalize( rdec ); rdec->range >>= 1; /* symbol <<= 1; */ /* if( rdec->code >= rdec->range ) { rdec->code -= rdec->range; symbol |= 1; } */- bit = ( rdec->code >= rdec->range );+ const bool bit = ( rdec->code >= rdec->range ); symbol <<= 1; symbol += bit; rdec->code -= rdec->range & ( 0U - bit ); }@@ -176,42 +174,75 @@ static inline unsigned Rd_decode_bit( struct Range_decoder * const rdec, Bit_model * const probability ) {- uint32_t bound; Rd_normalize( rdec );- bound = ( rdec->range >> bit_model_total_bits ) * *probability;+ const uint32_t bound = ( rdec->range >> bit_model_total_bits ) * *probability; if( rdec->code < bound ) {- *probability += (bit_model_total - *probability) >> bit_model_move_bits; rdec->range = bound;+ *probability += ( bit_model_total - *probability ) >> bit_model_move_bits; return 0; } else {- *probability -= *probability >> bit_model_move_bits; rdec->code -= bound; rdec->range -= bound;+ *probability -= *probability >> bit_model_move_bits; return 1; } } -static inline unsigned Rd_decode_tree3( struct Range_decoder * const rdec,- Bit_model bm[] )+static inline void Rd_decode_symbol_bit( struct Range_decoder * const rdec,+ Bit_model * const probability, unsigned * symbol ) {- unsigned symbol = 2 | Rd_decode_bit( rdec, &bm[1] );- symbol = ( symbol << 1 ) | Rd_decode_bit( rdec, &bm[symbol] );- symbol = ( symbol << 1 ) | Rd_decode_bit( rdec, &bm[symbol] );- return symbol & 7;+ Rd_normalize( rdec );+ *symbol <<= 1;+ const uint32_t bound = ( rdec->range >> bit_model_total_bits ) * *probability;+ if( rdec->code < bound )+ {+ rdec->range = bound;+ *probability += ( bit_model_total - *probability ) >> bit_model_move_bits;+ }+ else+ {+ rdec->code -= bound;+ rdec->range -= bound;+ *probability -= *probability >> bit_model_move_bits;+ *symbol |= 1;+ } } +static inline void Rd_decode_symbol_bit_reversed( struct Range_decoder * const rdec,+ Bit_model * const probability, unsigned * model,+ unsigned * symbol, const int i )+ {+ Rd_normalize( rdec );+ *model <<= 1;+ const uint32_t bound = ( rdec->range >> bit_model_total_bits ) * *probability;+ if( rdec->code < bound )+ {+ rdec->range = bound;+ *probability += ( bit_model_total - *probability ) >> bit_model_move_bits;+ }+ else+ {+ rdec->code -= bound;+ rdec->range -= bound;+ *probability -= *probability >> bit_model_move_bits;+ *model |= 1;+ *symbol |= 1 << i;+ }+ }+ static inline unsigned Rd_decode_tree6( struct Range_decoder * const rdec, Bit_model bm[] ) {- unsigned symbol = 2 | Rd_decode_bit( rdec, &bm[1] );- symbol = ( symbol << 1 ) | Rd_decode_bit( rdec, &bm[symbol] );- symbol = ( symbol << 1 ) | Rd_decode_bit( rdec, &bm[symbol] );- symbol = ( symbol << 1 ) | Rd_decode_bit( rdec, &bm[symbol] );- symbol = ( symbol << 1 ) | Rd_decode_bit( rdec, &bm[symbol] );- symbol = ( symbol << 1 ) | Rd_decode_bit( rdec, &bm[symbol] );+ unsigned symbol = 1;+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol ); return symbol & 0x3F; } @@ -219,9 +250,14 @@ Bit_model bm[] ) { unsigned symbol = 1;- int i;- for( i = 0; i < 8; ++i )- symbol = ( symbol << 1 ) | Rd_decode_bit( rdec, &bm[symbol] );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol ); return symbol & 0xFF; } @@ -233,21 +269,19 @@ unsigned symbol = 0; int i; for( i = 0; i < num_bits; ++i )- {- const unsigned bit = Rd_decode_bit( rdec, &bm[model] );- model <<= 1; model += bit;- symbol |= ( bit << i );- }+ Rd_decode_symbol_bit_reversed( rdec, &bm[model], &model, &symbol, i ); return symbol; } static inline unsigned Rd_decode_tree_reversed4( struct Range_decoder * const rdec, Bit_model bm[] ) {- unsigned symbol = Rd_decode_bit( rdec, &bm[1] );- symbol += Rd_decode_bit( rdec, &bm[2+symbol] ) << 1;- symbol += Rd_decode_bit( rdec, &bm[4+symbol] ) << 2;- symbol += Rd_decode_bit( rdec, &bm[8+symbol] ) << 3;+ unsigned model = 1;+ unsigned symbol = 0;+ Rd_decode_symbol_bit_reversed( rdec, &bm[model], &model, &symbol, 0 );+ Rd_decode_symbol_bit_reversed( rdec, &bm[model], &model, &symbol, 1 );+ Rd_decode_symbol_bit_reversed( rdec, &bm[model], &model, &symbol, 2 );+ Rd_decode_symbol_bit_reversed( rdec, &bm[model], &model, &symbol, 3 ); return symbol; } @@ -270,11 +304,24 @@ struct Len_model * const lm, const int pos_state ) {+ Bit_model * bm;+ unsigned mask, offset, symbol = 1;+ if( Rd_decode_bit( rdec, &lm->choice1 ) == 0 )- return Rd_decode_tree3( rdec, lm->bm_low[pos_state] );+ { bm = lm->bm_low[pos_state]; mask = 7; offset = 0; goto len3; } if( Rd_decode_bit( rdec, &lm->choice2 ) == 0 )- return len_low_symbols + Rd_decode_tree3( rdec, lm->bm_mid[pos_state] );- return len_low_symbols + len_mid_symbols + Rd_decode_tree8( rdec, lm->bm_high );+ { bm = lm->bm_mid[pos_state]; mask = 7; offset = len_low_symbols; goto len3; }+ bm = lm->bm_high; mask = 0xFF; offset = len_low_symbols + len_mid_symbols;+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+len3:+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ Rd_decode_symbol_bit( rdec, &bm[symbol], &symbol );+ return ( symbol & mask ) + min_match_len + offset; }
cbits/encoder.c view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided@@ -21,18 +21,7 @@ { int32_t * ptr0 = e->eb.mb.pos_array + ( e->eb.mb.cyclic_pos << 1 ); int32_t * ptr1 = ptr0 + 1;- int32_t * newptr;- int len = 0, len0 = 0, len1 = 0;- int maxlen = 3; /* only used if pairs != 0 */- int num_pairs = 0;- const int pos1 = e->eb.mb.pos + 1;- const int min_pos = ( e->eb.mb.pos > e->eb.mb.dictionary_size ) ?- e->eb.mb.pos - e->eb.mb.dictionary_size : 0;- const uint8_t * const data = Mb_ptr_to_current_pos( &e->eb.mb );- int count, key2, key3, key4, newpos1;- unsigned tmp; int len_limit = e->match_len_limit;- if( len_limit > Mb_available_bytes( &e->eb.mb ) ) { e->been_flushed = true;@@ -40,12 +29,18 @@ if( len_limit < 4 ) { *ptr0 = *ptr1 = 0; return 0; } } - tmp = crc32[data[0]] ^ data[1];- key2 = tmp & ( num_prev_positions2 - 1 );+ int maxlen = 3; /* only used if pairs != 0 */+ int num_pairs = 0;+ const int min_pos = ( e->eb.mb.pos > e->eb.mb.dictionary_size ) ?+ e->eb.mb.pos - e->eb.mb.dictionary_size : 0;+ const uint8_t * const data = Mb_ptr_to_current_pos( &e->eb.mb );++ unsigned tmp = crc32[data[0]] ^ data[1];+ const int key2 = tmp & ( num_prev_positions2 - 1 ); tmp ^= (unsigned)data[2] << 8;- key3 = num_prev_positions2 + ( tmp & ( num_prev_positions3 - 1 ) );- key4 = num_prev_positions2 + num_prev_positions3 +- ( ( tmp ^ ( crc32[data[3]] << 5 ) ) & e->eb.mb.key4_mask );+ const int key3 = num_prev_positions2 + ( tmp & ( num_prev_positions3 - 1 ) );+ const int key4 = num_prev_positions2 + num_prev_positions3 ++ ( ( tmp ^ ( crc32[data[3]] << 5 ) ) & e->eb.mb.key4_mask ); if( pairs ) {@@ -54,7 +49,7 @@ if( np2 > min_pos && e->eb.mb.buffer[np2-1] == data[0] ) { pairs[0].dis = e->eb.mb.pos - np2;- pairs[0].len = maxlen = 2;+ pairs[0].len = maxlen = 2 + ( np2 == np3 ); num_pairs = 1; } if( np2 != np3 && np3 > min_pos && e->eb.mb.buffer[np3-1] == data[0] )@@ -73,19 +68,22 @@ } } + const int pos1 = e->eb.mb.pos + 1; e->eb.mb.prev_positions[key2] = pos1; e->eb.mb.prev_positions[key3] = pos1;- newpos1 = e->eb.mb.prev_positions[key4];+ int newpos1 = e->eb.mb.prev_positions[key4]; e->eb.mb.prev_positions[key4] = pos1; + int len = 0, len0 = 0, len1 = 0;++ int count; for( count = e->cycles; ; ) {- int delta; if( newpos1 <= min_pos || --count < 0 ) { *ptr0 = *ptr1 = 0; break; } if( e->been_flushed ) len = 0;- delta = pos1 - newpos1;- newptr = e->eb.mb.pos_array ++ const int delta = pos1 - newpos1;+ int32_t * const newptr = e->eb.mb.pos_array + ( ( e->eb.mb.cyclic_pos - delta + ( (e->eb.mb.cyclic_pos >= delta) ? 0 : e->eb.mb.dictionary_size + 1 ) ) << 1 ); if( data[len-delta] == data[len] )@@ -140,7 +138,6 @@ for( len_state = 0; len_state < len_states; ++len_state ) { int * const dsp = e->dis_slot_prices[len_state];- int * const dp = e->dis_prices[len_state]; const Bit_model * const bmds = e->eb.bm_dis_slot[len_state]; int slot = 0; for( ; slot < end_dis_model; ++slot )@@ -149,6 +146,7 @@ dsp[slot] = price_symbol6( bmds, slot ) + (((( slot >> 1 ) - 1 ) - dis_align_bits ) << price_shift_bits ); + int * const dp = e->dis_prices[len_state]; for( dis = 0; dis < start_dis_model; ++dis ) dp[dis] = dsp[dis]; for( ; dis < modeled_distances; ++dis )@@ -157,7 +155,7 @@ } -/* Returns the number of bytes advanced (ahead).+/* Return the number of bytes advanced (ahead). trials[0]..trials[ahead-1] contain the steps to encode. ( trials[0].dis4 == -1 ) means literal. A match/rep longer or equal than match_len_limit finishes the sequence.@@ -166,9 +164,8 @@ const int reps[num_rep_distances], const State state ) {- int main_len, num_pairs, i, rep, num_trials, len;- int rep_index = 0, cur = 0;- int replens[num_rep_distances];+ int num_pairs, num_trials;+ int i, rep, len; if( e->pending_num_pairs > 0 ) /* from previous call */ {@@ -177,8 +174,10 @@ } else num_pairs = LZe_read_match_distances( e );- main_len = ( num_pairs > 0 ) ? e->pairs[num_pairs-1].len : 0;+ const int main_len = ( num_pairs > 0 ) ? e->pairs[num_pairs-1].len : 0; + int replens[num_rep_distances];+ int rep_index = 0; for( i = 0; i < num_rep_distances; ++i ) { replens[i] = Mb_true_match_len( &e->eb.mb, 0, reps[i] + 1 );@@ -200,7 +199,6 @@ return main_len; } - { const int pos_state = Mb_data_position( &e->eb.mb ) & pos_state_mask; const int match_price = price1( e->eb.bm_match[state][pos_state] ); const int rep_match_price = match_price + price1( e->eb.bm_rep[state] );@@ -238,9 +236,8 @@ for( rep = 0; rep < num_rep_distances; ++rep ) {- int price; if( replens[rep] < min_match_len ) continue;- price = rep_match_price + LZeb_price_rep( &e->eb, rep, state, pos_state );+ const int price = rep_match_price + LZeb_price_rep( &e->eb, rep, state, pos_state ); for( len = min_match_len; len <= replens[rep]; ++len ) Tr_update( &e->trials[len], price + Lp_price( &e->rep_len_prices, len, pos_state ), rep, 0 );@@ -260,17 +257,10 @@ if( ++len > e->pairs[i].len && ++i >= num_pairs ) break; } }- } + int cur = 0; while( true ) /* price optimization loop */ {- struct Trial *cur_trial, *next_trial;- int newlen, pos_state, triable_bytes, len_limit;- int start_len = min_match_len;- int next_price, match_price, rep_match_price;- State cur_state;- uint8_t prev_byte, cur_byte, match_byte;- if( !Mb_move_pos( &e->eb.mb ) ) return 0; if( ++cur >= num_trials ) /* no more initialized trials */ {@@ -278,8 +268,8 @@ return cur; } - num_pairs = LZe_read_match_distances( e );- newlen = ( num_pairs > 0 ) ? e->pairs[num_pairs-1].len : 0;+ const int num_pairs = LZe_read_match_distances( e );+ const int newlen = ( num_pairs > 0 ) ? e->pairs[num_pairs-1].len : 0; if( newlen >= e->match_len_limit ) { e->pending_num_pairs = num_pairs;@@ -288,7 +278,8 @@ } /* give final values to current trial */- cur_trial = &e->trials[cur];+ struct Trial * cur_trial = &e->trials[cur];+ State cur_state; { const int dis4 = cur_trial->dis4; int prev_index = cur_trial->prev_index;@@ -319,25 +310,25 @@ mtf_reps( dis4, cur_trial->reps ); /* literal is ignored */ } - pos_state = Mb_data_position( &e->eb.mb ) & pos_state_mask;- prev_byte = Mb_peek( &e->eb.mb, 1 );- cur_byte = Mb_peek( &e->eb.mb, 0 );- match_byte = Mb_peek( &e->eb.mb, cur_trial->reps[0] + 1 );+ const int pos_state = Mb_data_position( &e->eb.mb ) & pos_state_mask;+ const uint8_t prev_byte = Mb_peek( &e->eb.mb, 1 );+ const uint8_t cur_byte = Mb_peek( &e->eb.mb, 0 );+ const uint8_t match_byte = Mb_peek( &e->eb.mb, cur_trial->reps[0] + 1 ); - next_price = cur_trial->price +- price0( e->eb.bm_match[cur_state][pos_state] );+ int next_price = cur_trial->price ++ price0( e->eb.bm_match[cur_state][pos_state] ); if( St_is_char( cur_state ) ) next_price += LZeb_price_literal( &e->eb, prev_byte, cur_byte ); else next_price += LZeb_price_matched( &e->eb, prev_byte, cur_byte, match_byte ); /* try last updates to next trial */- next_trial = &e->trials[cur+1];+ struct Trial * next_trial = &e->trials[cur+1]; Tr_update( next_trial, next_price, -1, cur ); /* literal */ - match_price = cur_trial->price + price1( e->eb.bm_match[cur_state][pos_state] );- rep_match_price = match_price + price1( e->eb.bm_rep[cur_state] );+ const int match_price = cur_trial->price + price1( e->eb.bm_match[cur_state][pos_state] );+ const int rep_match_price = match_price + price1( e->eb.bm_rep[cur_state] ); if( match_byte == cur_byte && next_trial->dis4 != 0 && next_trial->prev_index2 == single_step_trial )@@ -352,11 +343,11 @@ } } - triable_bytes =+ const int triable_bytes = min( Mb_available_bytes( &e->eb.mb ), max_num_trials - 1 - cur ); if( triable_bytes < min_match_len ) continue; - len_limit = min( e->match_len_limit, triable_bytes );+ const int len_limit = min( e->match_len_limit, triable_bytes ); /* try literal + rep0 */ if( match_byte != cur_byte && next_trial->prev_index != cur )@@ -380,19 +371,20 @@ } } + int start_len = min_match_len;+ /* try rep distances */ for( rep = 0; rep < num_rep_distances; ++rep ) { const uint8_t * const data = Mb_ptr_to_current_pos( &e->eb.mb ); const int dis = cur_trial->reps[rep] + 1;- int price; if( data[0-dis] != data[0] || data[1-dis] != data[1] ) continue; for( len = min_match_len; len < len_limit; ++len ) if( data[len-dis] != data[len] ) break; while( num_trials < cur + len ) e->trials[++num_trials].price = infinite_price;- price = rep_match_price + LZeb_price_rep( &e->eb, rep, cur_state, pos_state );+ int price = rep_match_price + LZeb_price_rep( &e->eb, rep, cur_state, pos_state ); for( i = min_match_len; i <= len; ++i ) Tr_update( &e->trials[cur+i], price + Lp_price( &e->rep_len_prices, i, pos_state ), rep, cur );@@ -400,17 +392,14 @@ if( rep == 0 ) start_len = len + 1; /* discard shorter matches */ /* try rep + literal + rep0 */- { int len2 = len + 1; const int limit = min( e->match_len_limit + len2, triable_bytes );- int pos_state2;- State state2; while( len2 < limit && data[len2-dis] == data[len2] ) ++len2; len2 -= len + 1; if( len2 < min_match_len ) continue; - pos_state2 = ( pos_state + len ) & pos_state_mask;- state2 = St_set_rep( cur_state );+ int pos_state2 = ( pos_state + len ) & pos_state_mask;+ State state2 = St_set_rep( cur_state ); price += Lp_price( &e->rep_len_prices, len, pos_state ) + price0( e->eb.bm_match[state2][pos_state2] ) + LZeb_price_matched( &e->eb, data[len-1], data[len], data[len-dis] );@@ -423,21 +412,19 @@ e->trials[++num_trials].price = infinite_price; Tr_update3( &e->trials[cur+len+1+len2], price, rep, cur + len + 1, cur ); }- } /* try matches */ if( newlen >= start_len && newlen <= len_limit ) {- int dis; const int normal_match_price = match_price + price0( e->eb.bm_rep[cur_state] ); while( num_trials < cur + newlen ) e->trials[++num_trials].price = infinite_price; - i = 0;+ int i = 0; while( e->pairs[i].len < start_len ) ++i;- dis = e->pairs[i].dis;+ int dis = e->pairs[i].dis; for( len = start_len; ; ++len ) { int price = normal_match_price + LZe_price_pair( e, dis, len, pos_state );@@ -484,7 +471,7 @@ const int dis_price_count = best ? 1 : 512; const int align_price_count = best ? 1 : dis_align_size; const int price_count = ( e->match_len_limit > 36 ) ? 1013 : 4093;- int ahead, i;+ int i; State * const state = &e->eb.state; if( e->eb.member_finished ) return true;@@ -494,11 +481,10 @@ if( Mb_data_position( &e->eb.mb ) == 0 && !Mb_data_finished( &e->eb.mb ) ) /* encode first byte */ {- const uint8_t prev_byte = 0;- uint8_t cur_byte; if( !Mb_enough_available_bytes( &e->eb.mb ) || !Re_enough_free_bytes( &e->eb.renc ) ) return true;- cur_byte = Mb_peek( &e->eb.mb, 0 );+ const uint8_t prev_byte = 0;+ const uint8_t cur_byte = Mb_peek( &e->eb.mb, 0 ); Re_encode_bit( &e->eb.renc, &e->eb.bm_match[*state][0], 0 ); LZeb_encode_literal( &e->eb, prev_byte, cur_byte ); CRC32_update_byte( &e->eb.crc, cur_byte );@@ -525,7 +511,7 @@ Lp_update_prices( &e->rep_len_prices ); } - ahead = LZe_sequence_optimizer( e, e->eb.reps, *state );+ int ahead = LZe_sequence_optimizer( e, e->eb.reps, *state ); e->price_counter -= ahead; for( i = 0; ahead > 0; )@@ -542,14 +528,13 @@ const uint8_t prev_byte = Mb_peek( &e->eb.mb, ahead + 1 ); const uint8_t cur_byte = Mb_peek( &e->eb.mb, ahead ); CRC32_update_byte( &e->eb.crc, cur_byte );- if( St_is_char( *state ) )+ if( ( *state = St_set_char( *state ) ) < 4 ) LZeb_encode_literal( &e->eb, prev_byte, cur_byte ); else { const uint8_t match_byte = Mb_peek( &e->eb.mb, ahead + e->eb.reps[0] + 1 ); LZeb_encode_matched( &e->eb, prev_byte, cur_byte, match_byte ); }- *state = St_set_char( *state ); } else /* match or repeated match */ {
cbits/encoder.h view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided@@ -194,10 +194,9 @@ const int rep, const State state, const int pos_state ) {- int price; if( rep == 0 ) return price0( eb->bm_rep0[state] ) + price1( eb->bm_len[state][pos_state] );- price = price1( eb->bm_rep0[state] );+ int price = price1( eb->bm_rep0[state] ); if( rep == 1 ) price += price0( eb->bm_rep1[state] ); else
cbits/encoder_base.c view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided@@ -47,7 +47,6 @@ { const int buffer_size_limit = ( dict_factor * dict_size ) + before_size + after_size;- unsigned size; int i; mb->partial_data_pos = 0;@@ -66,9 +65,8 @@ mb->saved_dictionary_size = dict_size; mb->dictionary_size = dict_size; mb->pos_limit = mb->buffer_size - after_size;- size = 1 << max( 16, real_bits( mb->dictionary_size - 1 ) - 2 );- if( mb->dictionary_size > 1 << 26 ) /* 64 MiB */- size >>= 1;+ unsigned size = 1 << max( 16, real_bits( mb->dictionary_size - 1 ) - 2 );+ if( mb->dictionary_size > 1 << 26 ) size >>= 1; /* 64 MiB */ mb->key4_mask = size - 1; /* increases with dictionary size */ size += num_prev_positions23; mb->num_prev_positions = size;@@ -88,8 +86,7 @@ static void Mb_adjust_array( struct Matchfinder_base * const mb ) { int size = 1 << max( 16, real_bits( mb->dictionary_size - 1 ) - 2 );- if( mb->dictionary_size > 1 << 26 ) /* 64 MiB */- size >>= 1;+ if( mb->dictionary_size > 1 << 26 ) size >>= 1; /* 64 MiB */ mb->key4_mask = size - 1; size += mb->num_prev_positions23; mb->num_prev_positions = size;@@ -129,21 +126,21 @@ /* End Of Stream marker => (dis == 0xFFFFFFFFU, len == min_match_len) */ static void LZeb_try_full_flush( struct LZ_encoder_base * const eb ) {- int i;- const int pos_state = Mb_data_position( &eb->mb ) & pos_state_mask;- const State state = eb->state;- Lzip_trailer trailer; if( eb->member_finished || Cb_free_bytes( &eb->renc.cb ) < max_marker_size + eb->renc.ff_count + Lt_size ) return; eb->member_finished = true;+ const int pos_state = Mb_data_position( &eb->mb ) & pos_state_mask;+ const State state = eb->state; Re_encode_bit( &eb->renc, &eb->bm_match[state][pos_state], 1 ); Re_encode_bit( &eb->renc, &eb->bm_rep[state], 0 ); LZeb_encode_pair( eb, 0xFFFFFFFFU, min_match_len, pos_state ); Re_flush( &eb->renc );+ Lzip_trailer trailer; Lt_set_data_crc( trailer, LZeb_crc( eb ) ); Lt_set_data_size( trailer, Mb_data_position( &eb->mb ) ); Lt_set_member_size( trailer, Re_member_position( &eb->renc ) + Lt_size );+ int i; for( i = 0; i < Lt_size; ++i ) Cb_put_byte( &eb->renc.cb, trailer[i] ); }@@ -152,13 +149,13 @@ /* Sync Flush marker => (dis == 0xFFFFFFFFU, len == min_match_len + 1) */ static void LZeb_try_sync_flush( struct LZ_encoder_base * const eb ) {- const int pos_state = Mb_data_position( &eb->mb ) & pos_state_mask;- const State state = eb->state; const unsigned min_size = eb->renc.ff_count + max_marker_size; if( eb->member_finished || Cb_free_bytes( &eb->renc.cb ) < min_size + max_marker_size ) return; eb->mb.sync_flush_pending = false; const unsigned long long old_mpos = Re_member_position( &eb->renc );+ const int pos_state = Mb_data_position( &eb->mb ) & pos_state_mask;+ const State state = eb->state; do { /* size of markers must be >= rd_min_available_bytes + 5 */ Re_encode_bit( &eb->renc, &eb->bm_match[state][pos_state], 1 ); Re_encode_bit( &eb->renc, &eb->bm_rep[state], 0 );
cbits/encoder_base.h view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided@@ -147,10 +147,9 @@ static inline int price_symbol3( const Bit_model bm[], int symbol ) {- int price; bool bit = symbol & 1; symbol |= 8; symbol >>= 1;- price = price_bit( bm[symbol], bit );+ int price = price_bit( bm[symbol], bit ); bit = symbol & 1; symbol >>= 1; price += price_bit( bm[symbol], bit ); return price + price_bit( bm[1], symbol & 1 ); }@@ -158,10 +157,9 @@ static inline int price_symbol6( const Bit_model bm[], unsigned symbol ) {- int price; bool bit = symbol & 1; symbol |= 64; symbol >>= 1;- price = price_bit( bm[symbol], bit );+ int price = price_bit( bm[symbol], bit ); bit = symbol & 1; symbol >>= 1; price += price_bit( bm[symbol], bit ); bit = symbol & 1; symbol >>= 1; price += price_bit( bm[symbol], bit ); bit = symbol & 1; symbol >>= 1; price += price_bit( bm[symbol], bit );@@ -172,10 +170,9 @@ static inline int price_symbol8( const Bit_model bm[], int symbol ) {- int price; bool bit = symbol & 1; symbol |= 0x100; symbol >>= 1;- price = price_bit( bm[symbol], bit );+ int price = price_bit( bm[symbol], bit ); bit = symbol & 1; symbol >>= 1; price += price_bit( bm[symbol], bit ); bit = symbol & 1; symbol >>= 1; price += price_bit( bm[symbol], bit ); bit = symbol & 1; symbol >>= 1; price += price_bit( bm[symbol], bit );@@ -427,10 +424,9 @@ static inline void Re_encode_tree3( struct Range_encoder * const renc, Bit_model bm[], const int symbol ) {- int model; bool bit = ( symbol >> 2 ) & 1; Re_encode_bit( renc, &bm[1], bit );- model = 2 | bit;+ int model = 2 | bit; bit = ( symbol >> 1 ) & 1; Re_encode_bit( renc, &bm[model], bit ); model <<= 1; model |= bit; Re_encode_bit( renc, &bm[model], symbol & 1 );@@ -439,10 +435,9 @@ static inline void Re_encode_tree6( struct Range_encoder * const renc, Bit_model bm[], const unsigned symbol ) {- int model; bool bit = ( symbol >> 5 ) & 1; Re_encode_bit( renc, &bm[1], bit );- model = 2 | bit;+ int model = 2 | bit; bit = ( symbol >> 4 ) & 1; Re_encode_bit( renc, &bm[model], bit ); model <<= 1; model |= bit; bit = ( symbol >> 3 ) & 1;@@ -583,8 +578,7 @@ static inline void LZeb_encode_literal( struct LZ_encoder_base * const eb, const uint8_t prev_byte, const uint8_t symbol )- { Re_encode_tree8( &eb->renc, eb->bm_literal[get_lit_state(prev_byte)],- symbol ); }+ { Re_encode_tree8( &eb->renc, eb->bm_literal[get_lit_state(prev_byte)], symbol ); } static inline void LZeb_encode_matched( struct LZ_encoder_base * const eb, const uint8_t prev_byte, const uint8_t symbol, const uint8_t match_byte )@@ -595,8 +589,8 @@ const unsigned dis, const int len, const int pos_state ) {- const unsigned dis_slot = get_slot( dis ); Re_encode_len( &eb->renc, &eb->match_len_model, len, pos_state );+ const unsigned dis_slot = get_slot( dis ); Re_encode_tree6( &eb->renc, eb->bm_dis_slot[get_len_state(len)], dis_slot ); if( dis_slot >= start_dis_model )
cbits/fast_encoder.c view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided@@ -20,25 +20,24 @@ static int FLZe_longest_match_len( struct FLZ_encoder * const fe, int * const distance ) { enum { len_limit = 16 };- const uint8_t * const data = Mb_ptr_to_current_pos( &fe->eb.mb ); int32_t * ptr0 = fe->eb.mb.pos_array + fe->eb.mb.cyclic_pos;- const int pos1 = fe->eb.mb.pos + 1;- int maxlen = 0, newpos1, count; const int available = min( Mb_available_bytes( &fe->eb.mb ), max_match_len ); if( available < len_limit ) { *ptr0 = 0; return 0; } + const uint8_t * const data = Mb_ptr_to_current_pos( &fe->eb.mb ); fe->key4 = ( ( fe->key4 << 4 ) ^ data[3] ) & fe->eb.mb.key4_mask;- newpos1 = fe->eb.mb.prev_positions[fe->key4];+ const int pos1 = fe->eb.mb.pos + 1;+ int newpos1 = fe->eb.mb.prev_positions[fe->key4]; fe->eb.mb.prev_positions[fe->key4] = pos1;+ int maxlen = 0, count; for( count = 4; ; ) {- int32_t * newptr; int delta; if( newpos1 <= 0 || --count < 0 || ( delta = pos1 - newpos1 ) > fe->eb.mb.dictionary_size ) { *ptr0 = 0; break; }- newptr = fe->eb.mb.pos_array ++ int32_t * const newptr = fe->eb.mb.pos_array + ( fe->eb.mb.cyclic_pos - delta + ( ( fe->eb.mb.cyclic_pos >= delta ) ? 0 : fe->eb.mb.dictionary_size + 1 ) ); @@ -71,11 +70,10 @@ if( Mb_data_position( &fe->eb.mb ) == 0 && !Mb_data_finished( &fe->eb.mb ) ) /* encode first byte */ {- const uint8_t prev_byte = 0;- uint8_t cur_byte; if( !Mb_enough_available_bytes( &fe->eb.mb ) || !Re_enough_free_bytes( &fe->eb.renc ) ) return true;- cur_byte = Mb_peek( &fe->eb.mb, 0 );+ const uint8_t prev_byte = 0;+ const uint8_t cur_byte = Mb_peek( &fe->eb.mb, 0 ); Re_encode_bit( &fe->eb.renc, &fe->eb.bm_match[*state][0], 0 ); LZeb_encode_literal( &fe->eb, prev_byte, cur_byte ); CRC32_update_byte( &fe->eb.crc, cur_byte );@@ -86,13 +84,12 @@ while( !Mb_data_finished( &fe->eb.mb ) && Re_member_position( &fe->eb.renc ) < fe->eb.member_size_limit ) {- int match_distance = 0; /* avoid warning from gcc 6.1.0 */- int main_len, pos_state;- int len = 0; if( !Mb_enough_available_bytes( &fe->eb.mb ) || !Re_enough_free_bytes( &fe->eb.renc ) ) return true;- main_len = FLZe_longest_match_len( fe, &match_distance );- pos_state = Mb_data_position( &fe->eb.mb ) & pos_state_mask;+ int match_distance = 0; /* avoid warning from gcc 6.1.0 */+ const int main_len = FLZe_longest_match_len( fe, &match_distance );+ const int pos_state = Mb_data_position( &fe->eb.mb ) & pos_state_mask;+ int len = 0; for( i = 0; i < num_rep_distances; ++i ) {@@ -109,11 +106,10 @@ Re_encode_bit( &fe->eb.renc, &fe->eb.bm_len[*state][pos_state], 1 ); else {- int distance; Re_encode_bit( &fe->eb.renc, &fe->eb.bm_rep1[*state], rep > 1 ); if( rep > 1 ) Re_encode_bit( &fe->eb.renc, &fe->eb.bm_rep2[*state], rep > 2 );- distance = fe->eb.reps[rep];+ const int distance = fe->eb.reps[rep]; for( i = rep; i > 0; --i ) fe->eb.reps[i] = fe->eb.reps[i-1]; fe->eb.reps[0] = distance; }@@ -138,7 +134,6 @@ continue; } - { const uint8_t prev_byte = Mb_peek( &fe->eb.mb, 1 ); const uint8_t cur_byte = Mb_peek( &fe->eb.mb, 0 ); const uint8_t match_byte = Mb_peek( &fe->eb.mb, fe->eb.reps[0] + 1 );@@ -169,12 +164,10 @@ /* literal byte */ Re_encode_bit( &fe->eb.renc, &fe->eb.bm_match[*state][pos_state], 0 );- if( St_is_char( *state ) )+ if( ( *state = St_set_char( *state ) ) < 4 ) LZeb_encode_literal( &fe->eb, prev_byte, cur_byte ); else LZeb_encode_matched( &fe->eb, prev_byte, cur_byte, match_byte );- *state = St_set_char( *state );- } } LZeb_try_full_flush( &fe->eb );
cbits/fast_encoder.h view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided
cbits/lzip.h view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided@@ -168,6 +168,7 @@ static inline void CRC32_update_byte( uint32_t * const crc, const uint8_t byte ) { *crc = crc32[(*crc^byte)&0xFF] ^ ( *crc >> 8 ); } +/* about as fast as it is possible without messing with endianness */ static inline void CRC32_update_buf( uint32_t * const crc, const uint8_t * const buffer, const int size )
cbits/lzlib.c view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided
cbits/lzlib.h view
@@ -1,5 +1,5 @@ /* Lzlib - Compression library for the lzip format- Copyright (C) 2009-2021 Antonio Diaz Diaz.+ Copyright (C) 2009-2022 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided@@ -24,9 +24,9 @@ /* LZ_API_VERSION was first defined in lzlib 1.8 to 1. Since lzlib 1.12, LZ_API_VERSION is defined as (major * 1000 + minor). */ -#define LZ_API_VERSION 1012+#define LZ_API_VERSION 1013 -static const char * const LZ_version_string = "1.12";+static const char * const LZ_version_string = "1.13"; enum LZ_Errno { LZ_ok = 0, LZ_bad_argument, LZ_mem_error, LZ_sequence_error, LZ_header_error, LZ_unexpected_eof,
lzlib.cabal view
@@ -1,6 +1,6 @@ cabal-version: 1.18 name: lzlib-version: 1.0.7.1+version: 1.0.7.2 license: BSD3 license-file: LICENSE copyright: Copyright: (c) 2019-2021 Vanessa McHale@@ -53,7 +53,7 @@ ghc-options: -Wall build-depends: base >=4.7 && <5,- bytestring -any+ bytestring if impl(ghc >=8.0) ghc-options:@@ -76,12 +76,12 @@ default-language: Haskell2010 ghc-options: -threaded -rtsopts "-with-rtsopts=-N -K1K" -Wall build-depends:- base -any,- lzlib -any,- hspec -any,- bytestring -any,- directory -any,- pathological-bytestrings -any+ base,+ lzlib,+ hspec,+ bytestring,+ directory,+ pathological-bytestrings if impl(ghc >=8.0) ghc-options:@@ -101,12 +101,12 @@ default-language: Haskell2010 ghc-options: -Wall build-depends:- base -any,- lzlib -any,- criterion -any,- bytestring -any,- temporary -any,- filepath -any+ base,+ lzlib,+ criterion,+ bytestring,+ temporary,+ filepath if impl(ghc >=8.0) ghc-options:@@ -126,11 +126,11 @@ default-language: Haskell2010 ghc-options: -threaded -rtsopts -with-rtsopts=-N -Wall build-depends:- base -any,- lzlib -any,- filepath -any,- temporary -any,- bytestring -any+ base,+ lzlib,+ filepath,+ temporary,+ bytestring if impl(ghc >=8.0) ghc-options: -Wincomplete-uni-patterns -Wincomplete-record-updates
mem/Mem.hs view
@@ -14,13 +14,12 @@ decompressDump :: IO () decompressDump = withSystemTempDirectory "lz" $- \fp -> BSL.writeFile (fp </> "gmp-6.1.2.tar") =<<- (decompress <$> BSL.readFile "gmp-6.1.2.tar.lz")+ \fp -> BSL.writeFile (fp </> "gmp-6.1.2.tar") . decompress =<< BSL.readFile "gmp-6.1.2.tar.lz" compressDump :: IO () compressDump = withSystemTempDirectory "lz" $ \fp -> BSL.writeFile (fp </> "gmp-6.1.2.tar.lz") =<<- (compressFile "gmp-6.1.2.tar")+ compressFile "gmp-6.1.2.tar" forceHead :: BSL.ByteString -> IO () forceHead bsl = BS.length (head $ BSL.toChunks bsl) `seq` mempty
test/Spec.hs view
@@ -55,7 +55,9 @@ main :: IO () main = do ex' <- filterM doesFileExist ["gmp-6.1.2.tar.lz", "lzlib-1.10.tar.lz"]- ex <- filterM doesFileExist ["cbits/encoder.c", "gmp-6.1.2.tar", "lzlib-1.10.tar"]+ ex <- ("test/data/7341e7190c2219dbc2641707c5c0e41e038f8a44.jpg" :)+ . ("cbits/encoder.c" :)+ <$> filterM doesFileExist ["gmp-6.1.2.tar", "lzlib-1.10.tar"] hspec $ do describe "roundtrip" $ traverse_ compressFile ex