lz4-frame-conduit (empty) → 0.1.0.0
raw patch · 16 files changed
+10222/−0 lines, 16 filesdep +QuickCheckdep +basedep +bytestring
Dependencies added: QuickCheck, base, bytestring, conduit, conduit-extra, containers, hspec, inline-c, lz4-frame-conduit, optparse-applicative, raw-strings-qq, resourcet, template-haskell, text, unliftio, unliftio-core
Files
- LICENSE +21/−0
- README.md +20/−0
- app/Main.hs +64/−0
- lz4-frame-conduit.cabal +87/−0
- lz4/lib/lz4.c +2398/−0
- lz4/lib/lz4.h +764/−0
- lz4/lib/lz4frame.c +1860/−0
- lz4/lib/lz4frame.h +615/−0
- lz4/lib/lz4frame_static.h +47/−0
- lz4/lib/lz4hc.c +1538/−0
- lz4/lib/lz4hc.h +438/−0
- lz4/lib/xxhash.c +1030/−0
- lz4/lib/xxhash.h +328/−0
- src/Codec/Compression/LZ4/CTypes.hsc +239/−0
- src/Codec/Compression/LZ4/Conduit.hsc +692/−0
- test/Main.hs +81/−0
+ LICENSE view
@@ -0,0 +1,21 @@+The MIT License++Copyright (c) 2017 Niklas Hambüchen++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.
+ README.md view
@@ -0,0 +1,20 @@+# lz4-frame-conduit++Haskell Conduit implementing the official LZ4 frame streaming format.++## Rationale and comparison to non-`lz4`-compatible libraries++There exist two `lz4` formats:++1. the **block format**, limited to compressing data < 2 GB+2. the **frame format**, suitable for streaming arbitrarily sized data+ ** This is what the `lz4` command line utility uses++**This library implements the frame format.**++Some existing Haskell libraries implement only the block format, and are thus not suitable to compress data > 2 GB in a way.+(Of course they could chunk the output in some arbitrary way, but that wouldn't be compatible with the `lz4` command line utility.)+The libraries that belong to this category at time of writing are:++* [`lz4`](https://hackage.haskell.org/package/lz4)+* [`lz4-conduit`](https://hackage.haskell.org/package/lz4-conduit)
+ app/Main.hs view
@@ -0,0 +1,64 @@+{-# LANGUAGE NamedFieldPuns #-}+{-# LANGUAGE OverloadedStrings #-}++module Main (main) where++import qualified Codec.Compression.LZ4.Conduit as LZ4F+import Control.Applicative (optional)+import Control.Monad.Trans.Resource (runResourceT)+import Data.Conduit+import qualified Data.Conduit.Binary as CB+import Data.Monoid ((<>))+import Data.Text (Text)+import qualified Data.Text as T+import Options.Applicative (Parser, argument, str, metavar, long, short)+import qualified Options.Applicative as Opts+import System.IO (stdin, stdout)+++-- | Command line arguments of this program.+data CLIArgs = CLIArgs+ { m'inputFile :: Maybe Text+ , m'outputFile :: Maybe Text+ , decompress :: Bool+ } deriving (Eq, Ord, Show)+++cliArgsParser :: Parser CLIArgs+cliArgsParser = CLIArgs+ <$> optional (T.pack <$> argument str (metavar "INPUT_FILE"))+ <*> optional (T.pack <$> argument str (metavar "OUTPUT-FILE"))+ <*> Opts.switch (long "decompress" <> short 'd')+++-- | Parses the command line arguments for this program.+parseArgs :: IO CLIArgs+parseArgs = Opts.execParser $+ Opts.info+ (Opts.helper <*> cliArgsParser)+ (Opts.fullDesc <> Opts.progDesc "Compress or decompress .lz4 files")+++main :: IO ()+main = do+ CLIArgs+ { m'inputFile+ , m'outputFile+ , decompress+ } <- parseArgs++ let source = case m'inputFile of+ Nothing -> CB.sourceHandle stdin+ Just "-" -> CB.sourceHandle stdin+ Just path -> CB.sourceFile (T.unpack path)++ let sink = case m'outputFile of+ Nothing -> CB.sinkHandle stdout+ Just "-" -> CB.sinkHandle stdout+ Just path -> CB.sinkFile (T.unpack path)++ let compressOrDecompress+ | decompress = LZ4F.decompress+ | otherwise = LZ4F.compress++ runResourceT $ runConduit $ source .| compressOrDecompress .| sink
+ lz4-frame-conduit.cabal view
@@ -0,0 +1,87 @@+name: lz4-frame-conduit+version: 0.1.0.0+synopsis: Conduit implementing the official LZ4 frame streaming format+description:+ Conduit implementing the official LZ4 frame streaming format.+ Compatible with the lz4 command line utility.+homepage: https://github.com/nh2/lz4-frame-conduit#readme+bug-reports: https://github.com/nh2/lz4-frame-conduit/issues+license: MIT+license-file: LICENSE+author: Niklas Hambüchen+maintainer: mail@nh2.me+copyright: 2017 Niklas Hambüchen+category: Web+build-type: Simple+cabal-version: >=1.10++extra-source-files: README.md+ lz4/lib/lz4.c+ lz4/lib/lz4.h+ lz4/lib/lz4frame.c+ lz4/lib/lz4frame.h+ lz4/lib/lz4frame_static.h+ lz4/lib/lz4hc.c+ lz4/lib/lz4hc.h+ lz4/lib/xxhash.c+ lz4/lib/xxhash.h+++library+ hs-source-dirs: src+ exposed-modules: Codec.Compression.LZ4.Conduit+ Codec.Compression.LZ4.CTypes+ build-depends: base >= 4.7 && < 5+ , conduit+ , conduit-extra+ , containers+ , bytestring+ , inline-c+ , raw-strings-qq+ , resourcet+ , template-haskell+ , unliftio+ , unliftio-core+ build-tools: hsc2hs+ include-dirs: lz4/lib+ c-sources: lz4/lib/lz4.c+ lz4/lib/lz4frame.c+ lz4/lib/lz4hc.c+ lz4/lib/xxhash.c+ other-extensions: TemplateHaskell+ ghc-options: -Wall+ default-language: Haskell2010++executable haskell-lz4c+ hs-source-dirs: app+ main-is: Main.hs+ build-depends: base+ , lz4-frame-conduit+ , bytestring+ , conduit+ , conduit-extra+ , optparse-applicative+ , resourcet+ , text+ ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N+ default-language: Haskell2010++test-suite lz4-frame-conduit-test+ type: exitcode-stdio-1.0+ hs-source-dirs: test+ main-is: Main.hs+ build-depends: base+ , lz4-frame-conduit+ , bytestring+ , conduit+ , conduit-extra+ , hspec+ , resourcet+ , unliftio-core+ , QuickCheck+ ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N+ default-language: Haskell2010++source-repository head+ type: git+ location: https://github.com/nh2/lz4-frame-conduit
+ lz4/lib/lz4.c view
@@ -0,0 +1,2398 @@+/*+ LZ4 - Fast LZ compression algorithm+ Copyright (C) 2011-present, Yann Collet.++ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)++ Redistribution and use in source and binary forms, with or without+ modification, are permitted provided that the following conditions are+ met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.+ * Redistributions in binary form must reproduce the above+ copyright notice, this list of conditions and the following disclaimer+ in the documentation and/or other materials provided with the+ distribution.++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.++ You can contact the author at :+ - LZ4 homepage : http://www.lz4.org+ - LZ4 source repository : https://github.com/lz4/lz4+*/++/*-************************************+* Tuning parameters+**************************************/+/*+ * LZ4_HEAPMODE :+ * Select how default compression functions will allocate memory for their hash table,+ * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).+ */+#ifndef LZ4_HEAPMODE+# define LZ4_HEAPMODE 0+#endif++/*+ * ACCELERATION_DEFAULT :+ * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0+ */+#define ACCELERATION_DEFAULT 1+++/*-************************************+* CPU Feature Detection+**************************************/+/* LZ4_FORCE_MEMORY_ACCESS+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.+ * The below switch allow to select different access method for improved performance.+ * Method 0 (default) : use `memcpy()`. Safe and portable.+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).+ * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.+ * Method 2 : direct access. This method is portable but violate C standard.+ * It can generate buggy code on targets which assembly generation depends on alignment.+ * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)+ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.+ * Prefer these methods in priority order (0 > 1 > 2)+ */+#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */+# if defined(__GNUC__) && \+ ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \+ || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )+# define LZ4_FORCE_MEMORY_ACCESS 2+# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__)+# define LZ4_FORCE_MEMORY_ACCESS 1+# endif+#endif++/*+ * LZ4_FORCE_SW_BITCOUNT+ * Define this parameter if your target system or compiler does not support hardware bit count+ */+#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */+# define LZ4_FORCE_SW_BITCOUNT+#endif++++/*-************************************+* Dependency+**************************************/+/*+ * LZ4_SRC_INCLUDED:+ * Amalgamation flag, whether lz4.c is included+ */+#ifndef LZ4_SRC_INCLUDED+# define LZ4_SRC_INCLUDED 1+#endif++#ifndef LZ4_STATIC_LINKING_ONLY+#define LZ4_STATIC_LINKING_ONLY+#endif++#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS+#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */+#endif++#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */+#include "lz4.h"+/* see also "memory routines" below */+++/*-************************************+* Compiler Options+**************************************/+#ifdef _MSC_VER /* Visual Studio */+# include <intrin.h>+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */+# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */+#endif /* _MSC_VER */++#ifndef LZ4_FORCE_INLINE+# ifdef _MSC_VER /* Visual Studio */+# define LZ4_FORCE_INLINE static __forceinline+# else+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */+# ifdef __GNUC__+# define LZ4_FORCE_INLINE static inline __attribute__((always_inline))+# else+# define LZ4_FORCE_INLINE static inline+# endif+# else+# define LZ4_FORCE_INLINE static+# endif /* __STDC_VERSION__ */+# endif /* _MSC_VER */+#endif /* LZ4_FORCE_INLINE */++/* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE+ * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8,+ * together with a simple 8-byte copy loop as a fall-back path.+ * However, this optimization hurts the decompression speed by >30%,+ * because the execution does not go to the optimized loop+ * for typical compressible data, and all of the preamble checks+ * before going to the fall-back path become useless overhead.+ * This optimization happens only with the -O3 flag, and -O2 generates+ * a simple 8-byte copy loop.+ * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8+ * functions are annotated with __attribute__((optimize("O2"))),+ * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute+ * of LZ4_wildCopy8 does not affect the compression speed.+ */+#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__)+# define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2")))+# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE+#else+# define LZ4_FORCE_O2_GCC_PPC64LE+# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static+#endif++#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__)+# define expect(expr,value) (__builtin_expect ((expr),(value)) )+#else+# define expect(expr,value) (expr)+#endif++#ifndef likely+#define likely(expr) expect((expr) != 0, 1)+#endif+#ifndef unlikely+#define unlikely(expr) expect((expr) != 0, 0)+#endif+++/*-************************************+* Memory routines+**************************************/+#include <stdlib.h> /* malloc, calloc, free */+#define ALLOC(s) malloc(s)+#define ALLOC_AND_ZERO(s) calloc(1,s)+#define FREEMEM(p) free(p)+#include <string.h> /* memset, memcpy */+#define MEM_INIT(p,v,s) memset((p),(v),(s))+++/*-************************************+* Common Constants+**************************************/+#define MINMATCH 4++#define WILDCOPYLENGTH 8+#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */+#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */+#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */+#define FASTLOOP_SAFE_DISTANCE 64+static const int LZ4_minLength = (MFLIMIT+1);++#define KB *(1 <<10)+#define MB *(1 <<20)+#define GB *(1U<<30)++#define LZ4_DISTANCE_ABSOLUTE_MAX 65535+#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */+# error "LZ4_DISTANCE_MAX is too big : must be <= 65535"+#endif++#define ML_BITS 4+#define ML_MASK ((1U<<ML_BITS)-1)+#define RUN_BITS (8-ML_BITS)+#define RUN_MASK ((1U<<RUN_BITS)-1)+++/*-************************************+* Error detection+**************************************/+#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)+# include <assert.h>+#else+# ifndef assert+# define assert(condition) ((void)0)+# endif+#endif++#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */++#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)+# include <stdio.h>+static int g_debuglog_enable = 1;+# define DEBUGLOG(l, ...) { \+ if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \+ fprintf(stderr, __FILE__ ": "); \+ fprintf(stderr, __VA_ARGS__); \+ fprintf(stderr, " \n"); \+ } }+#else+# define DEBUGLOG(l, ...) {} /* disabled */+#endif+++/*-************************************+* Types+**************************************/+#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)+# include <stdint.h>+ typedef uint8_t BYTE;+ typedef uint16_t U16;+ typedef uint32_t U32;+ typedef int32_t S32;+ typedef uint64_t U64;+ typedef uintptr_t uptrval;+#else+ typedef unsigned char BYTE;+ typedef unsigned short U16;+ typedef unsigned int U32;+ typedef signed int S32;+ typedef unsigned long long U64;+ typedef size_t uptrval; /* generally true, except OpenVMS-64 */+#endif++#if defined(__x86_64__)+ typedef U64 reg_t; /* 64-bits in x32 mode */+#else+ typedef size_t reg_t; /* 32-bits in x32 mode */+#endif++typedef enum {+ notLimited = 0,+ limitedOutput = 1,+ fillOutput = 2+} limitedOutput_directive;+++/*-************************************+* Reading and writing into memory+**************************************/+static unsigned LZ4_isLittleEndian(void)+{+ const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */+ return one.c[0];+}+++#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2)+/* lie to the compiler about data alignment; use with caution */++static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; }+static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; }+static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; }++static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }+static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }++#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1)++/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */+/* currently only defined for gcc and icc */+typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign;++static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }+static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }+static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; }++static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }+static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }++#else /* safe and portable access using memcpy() */++static U16 LZ4_read16(const void* memPtr)+{+ U16 val; memcpy(&val, memPtr, sizeof(val)); return val;+}++static U32 LZ4_read32(const void* memPtr)+{+ U32 val; memcpy(&val, memPtr, sizeof(val)); return val;+}++static reg_t LZ4_read_ARCH(const void* memPtr)+{+ reg_t val; memcpy(&val, memPtr, sizeof(val)); return val;+}++static void LZ4_write16(void* memPtr, U16 value)+{+ memcpy(memPtr, &value, sizeof(value));+}++static void LZ4_write32(void* memPtr, U32 value)+{+ memcpy(memPtr, &value, sizeof(value));+}++#endif /* LZ4_FORCE_MEMORY_ACCESS */+++static U16 LZ4_readLE16(const void* memPtr)+{+ if (LZ4_isLittleEndian()) {+ return LZ4_read16(memPtr);+ } else {+ const BYTE* p = (const BYTE*)memPtr;+ return (U16)((U16)p[0] + (p[1]<<8));+ }+}++static void LZ4_writeLE16(void* memPtr, U16 value)+{+ if (LZ4_isLittleEndian()) {+ LZ4_write16(memPtr, value);+ } else {+ BYTE* p = (BYTE*)memPtr;+ p[0] = (BYTE) value;+ p[1] = (BYTE)(value>>8);+ }+}++/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */+LZ4_FORCE_O2_INLINE_GCC_PPC64LE+void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd)+{+ BYTE* d = (BYTE*)dstPtr;+ const BYTE* s = (const BYTE*)srcPtr;+ BYTE* const e = (BYTE*)dstEnd;++ do { memcpy(d,s,8); d+=8; s+=8; } while (d<e);+}++static const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4};+static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3};+++#ifndef LZ4_FAST_DEC_LOOP+# if defined(__i386__) || defined(__x86_64__)+# define LZ4_FAST_DEC_LOOP 1+# elif defined(__aarch64__) && !defined(__clang__)+ /* On aarch64, we disable this optimization for clang because on certain+ * mobile chipsets and clang, it reduces performance. For more information+ * refer to https://github.com/lz4/lz4/pull/707. */+# define LZ4_FAST_DEC_LOOP 1+# else+# define LZ4_FAST_DEC_LOOP 0+# endif+#endif++#if LZ4_FAST_DEC_LOOP++LZ4_FORCE_O2_INLINE_GCC_PPC64LE void+LZ4_memcpy_using_offset_base(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset)+{+ if (offset < 8) {+ dstPtr[0] = srcPtr[0];+ dstPtr[1] = srcPtr[1];+ dstPtr[2] = srcPtr[2];+ dstPtr[3] = srcPtr[3];+ srcPtr += inc32table[offset];+ memcpy(dstPtr+4, srcPtr, 4);+ srcPtr -= dec64table[offset];+ dstPtr += 8;+ } else {+ memcpy(dstPtr, srcPtr, 8);+ dstPtr += 8;+ srcPtr += 8;+ }++ LZ4_wildCopy8(dstPtr, srcPtr, dstEnd);+}++/* customized variant of memcpy, which can overwrite up to 32 bytes beyond dstEnd+ * this version copies two times 16 bytes (instead of one time 32 bytes)+ * because it must be compatible with offsets >= 16. */+LZ4_FORCE_O2_INLINE_GCC_PPC64LE void+LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd)+{+ BYTE* d = (BYTE*)dstPtr;+ const BYTE* s = (const BYTE*)srcPtr;+ BYTE* const e = (BYTE*)dstEnd;++ do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d<e);+}++/* LZ4_memcpy_using_offset() presumes :+ * - dstEnd >= dstPtr + MINMATCH+ * - there is at least 8 bytes available to write after dstEnd */+LZ4_FORCE_O2_INLINE_GCC_PPC64LE void+LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset)+{+ BYTE v[8];++ assert(dstEnd >= dstPtr + MINMATCH);+ LZ4_write32(dstPtr, 0); /* silence an msan warning when offset==0 */++ switch(offset) {+ case 1:+ memset(v, *srcPtr, 8);+ break;+ case 2:+ memcpy(v, srcPtr, 2);+ memcpy(&v[2], srcPtr, 2);+ memcpy(&v[4], &v[0], 4);+ break;+ case 4:+ memcpy(v, srcPtr, 4);+ memcpy(&v[4], srcPtr, 4);+ break;+ default:+ LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset);+ return;+ }++ memcpy(dstPtr, v, 8);+ dstPtr += 8;+ while (dstPtr < dstEnd) {+ memcpy(dstPtr, v, 8);+ dstPtr += 8;+ }+}+#endif+++/*-************************************+* Common functions+**************************************/+static unsigned LZ4_NbCommonBytes (reg_t val)+{+ if (LZ4_isLittleEndian()) {+ if (sizeof(val)==8) {+# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)+ unsigned long r = 0;+ _BitScanForward64( &r, (U64)val );+ return (int)(r>>3);+# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)+ return (unsigned)__builtin_ctzll((U64)val) >> 3;+# else+ static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,+ 0, 3, 1, 3, 1, 4, 2, 7,+ 0, 2, 3, 6, 1, 5, 3, 5,+ 1, 3, 4, 4, 2, 5, 6, 7,+ 7, 0, 1, 2, 3, 3, 4, 6,+ 2, 6, 5, 5, 3, 4, 5, 6,+ 7, 1, 2, 4, 6, 4, 4, 5,+ 7, 2, 6, 5, 7, 6, 7, 7 };+ return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];+# endif+ } else /* 32 bits */ {+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)+ unsigned long r;+ _BitScanForward( &r, (U32)val );+ return (int)(r>>3);+# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)+ return (unsigned)__builtin_ctz((U32)val) >> 3;+# else+ static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,+ 3, 2, 2, 1, 3, 2, 0, 1,+ 3, 3, 1, 2, 2, 2, 2, 0,+ 3, 1, 2, 0, 1, 0, 1, 1 };+ return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];+# endif+ }+ } else /* Big Endian CPU */ {+ if (sizeof(val)==8) { /* 64-bits */+# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)+ unsigned long r = 0;+ _BitScanReverse64( &r, val );+ return (unsigned)(r>>3);+# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)+ return (unsigned)__builtin_clzll((U64)val) >> 3;+# else+ static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits.+ Just to avoid some static analyzer complaining about shift by 32 on 32-bits target.+ Note that this code path is never triggered in 32-bits mode. */+ unsigned r;+ if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; }+ if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }+ r += (!val);+ return r;+# endif+ } else /* 32 bits */ {+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)+ unsigned long r = 0;+ _BitScanReverse( &r, (unsigned long)val );+ return (unsigned)(r>>3);+# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)+ return (unsigned)__builtin_clz((U32)val) >> 3;+# else+ unsigned r;+ if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }+ r += (!val);+ return r;+# endif+ }+ }+}++#define STEPSIZE sizeof(reg_t)+LZ4_FORCE_INLINE+unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)+{+ const BYTE* const pStart = pIn;++ if (likely(pIn < pInLimit-(STEPSIZE-1))) {+ reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn);+ if (!diff) {+ pIn+=STEPSIZE; pMatch+=STEPSIZE;+ } else {+ return LZ4_NbCommonBytes(diff);+ } }++ while (likely(pIn < pInLimit-(STEPSIZE-1))) {+ reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn);+ if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; }+ pIn += LZ4_NbCommonBytes(diff);+ return (unsigned)(pIn - pStart);+ }++ if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; }+ if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; }+ if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;+ return (unsigned)(pIn - pStart);+}+++#ifndef LZ4_COMMONDEFS_ONLY+/*-************************************+* Local Constants+**************************************/+static const int LZ4_64Klimit = ((64 KB) + (MFLIMIT-1));+static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */+++/*-************************************+* Local Structures and types+**************************************/+typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t;++/**+ * This enum distinguishes several different modes of accessing previous+ * content in the stream.+ *+ * - noDict : There is no preceding content.+ * - withPrefix64k : Table entries up to ctx->dictSize before the current blob+ * blob being compressed are valid and refer to the preceding+ * content (of length ctx->dictSize), which is available+ * contiguously preceding in memory the content currently+ * being compressed.+ * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere+ * else in memory, starting at ctx->dictionary with length+ * ctx->dictSize.+ * - usingDictCtx : Like usingExtDict, but everything concerning the preceding+ * content is in a separate context, pointed to by+ * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table+ * entries in the current context that refer to positions+ * preceding the beginning of the current compression are+ * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx+ * ->dictSize describe the location and size of the preceding+ * content, and matches are found by looking in the ctx+ * ->dictCtx->hashTable.+ */+typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive;+typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;+++/*-************************************+* Local Utils+**************************************/+int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; }+const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; }+int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }+int LZ4_sizeofState() { return LZ4_STREAMSIZE; }+++/*-************************************+* Internal Definitions used in Tests+**************************************/+#if defined (__cplusplus)+extern "C" {+#endif++int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize);++int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,+ int compressedSize, int maxOutputSize,+ const void* dictStart, size_t dictSize);++#if defined (__cplusplus)+}+#endif++/*-******************************+* Compression functions+********************************/+static U32 LZ4_hash4(U32 sequence, tableType_t const tableType)+{+ if (tableType == byU16)+ return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));+ else+ return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));+}++static U32 LZ4_hash5(U64 sequence, tableType_t const tableType)+{+ const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG;+ if (LZ4_isLittleEndian()) {+ const U64 prime5bytes = 889523592379ULL;+ return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));+ } else {+ const U64 prime8bytes = 11400714785074694791ULL;+ return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));+ }+}++LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType)+{+ if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType);+ return LZ4_hash4(LZ4_read32(p), tableType);+}++static void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType)+{+ switch (tableType)+ {+ default: /* fallthrough */+ case clearedTable: { /* illegal! */ assert(0); return; }+ case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; }+ case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; }+ case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; }+ }+}++static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType)+{+ switch (tableType)+ {+ default: /* fallthrough */+ case clearedTable: /* fallthrough */+ case byPtr: { /* illegal! */ assert(0); return; }+ case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; }+ case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; }+ }+}++static void LZ4_putPositionOnHash(const BYTE* p, U32 h,+ void* tableBase, tableType_t const tableType,+ const BYTE* srcBase)+{+ switch (tableType)+ {+ case clearedTable: { /* illegal! */ assert(0); return; }+ case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; }+ case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; }+ case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; }+ }+}++LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)+{+ U32 const h = LZ4_hashPosition(p, tableType);+ LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);+}++/* LZ4_getIndexOnHash() :+ * Index of match position registered in hash table.+ * hash position must be calculated by using base+index, or dictBase+index.+ * Assumption 1 : only valid if tableType == byU32 or byU16.+ * Assumption 2 : h is presumed valid (within limits of hash table)+ */+static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType)+{+ LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2);+ if (tableType == byU32) {+ const U32* const hashTable = (const U32*) tableBase;+ assert(h < (1U << (LZ4_MEMORY_USAGE-2)));+ return hashTable[h];+ }+ if (tableType == byU16) {+ const U16* const hashTable = (const U16*) tableBase;+ assert(h < (1U << (LZ4_MEMORY_USAGE-1)));+ return hashTable[h];+ }+ assert(0); return 0; /* forbidden case */+}++static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase)+{+ if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; }+ if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; return hashTable[h] + srcBase; }+ { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */+}++LZ4_FORCE_INLINE const BYTE*+LZ4_getPosition(const BYTE* p,+ const void* tableBase, tableType_t tableType,+ const BYTE* srcBase)+{+ U32 const h = LZ4_hashPosition(p, tableType);+ return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);+}++LZ4_FORCE_INLINE void+LZ4_prepareTable(LZ4_stream_t_internal* const cctx,+ const int inputSize,+ const tableType_t tableType) {+ /* If compression failed during the previous step, then the context+ * is marked as dirty, therefore, it has to be fully reset.+ */+ if (cctx->dirty) {+ DEBUGLOG(5, "LZ4_prepareTable: Full reset for %p", cctx);+ MEM_INIT(cctx, 0, sizeof(LZ4_stream_t_internal));+ return;+ }++ /* If the table hasn't been used, it's guaranteed to be zeroed out, and is+ * therefore safe to use no matter what mode we're in. Otherwise, we figure+ * out if it's safe to leave as is or whether it needs to be reset.+ */+ if (cctx->tableType != clearedTable) {+ assert(inputSize >= 0);+ if (cctx->tableType != tableType+ || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU)+ || ((tableType == byU32) && cctx->currentOffset > 1 GB)+ || tableType == byPtr+ || inputSize >= 4 KB)+ {+ DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx);+ MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);+ cctx->currentOffset = 0;+ cctx->tableType = clearedTable;+ } else {+ DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)");+ }+ }++ /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, is faster+ * than compressing without a gap. However, compressing with+ * currentOffset == 0 is faster still, so we preserve that case.+ */+ if (cctx->currentOffset != 0 && tableType == byU32) {+ DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset");+ cctx->currentOffset += 64 KB;+ }++ /* Finally, clear history */+ cctx->dictCtx = NULL;+ cctx->dictionary = NULL;+ cctx->dictSize = 0;+}++/** LZ4_compress_generic() :+ inlined, to ensure branches are decided at compilation time */+LZ4_FORCE_INLINE int LZ4_compress_generic(+ LZ4_stream_t_internal* const cctx,+ const char* const source,+ char* const dest,+ const int inputSize,+ int *inputConsumed, /* only written when outputDirective == fillOutput */+ const int maxOutputSize,+ const limitedOutput_directive outputDirective,+ const tableType_t tableType,+ const dict_directive dictDirective,+ const dictIssue_directive dictIssue,+ const int acceleration)+{+ int result;+ const BYTE* ip = (const BYTE*) source;++ U32 const startIndex = cctx->currentOffset;+ const BYTE* base = (const BYTE*) source - startIndex;+ const BYTE* lowLimit;++ const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;+ const BYTE* const dictionary =+ dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;+ const U32 dictSize =+ dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;+ const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */++ int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);+ U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */+ const BYTE* const dictEnd = dictionary + dictSize;+ const BYTE* anchor = (const BYTE*) source;+ const BYTE* const iend = ip + inputSize;+ const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1;+ const BYTE* const matchlimit = iend - LASTLITERALS;++ /* the dictCtx currentOffset is indexed on the start of the dictionary,+ * while a dictionary in the current context precedes the currentOffset */+ const BYTE* dictBase = (dictDirective == usingDictCtx) ?+ dictionary + dictSize - dictCtx->currentOffset :+ dictionary + dictSize - startIndex;++ BYTE* op = (BYTE*) dest;+ BYTE* const olimit = op + maxOutputSize;++ U32 offset = 0;+ U32 forwardH;++ DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType);+ /* If init conditions are not met, we don't have to mark stream+ * as having dirty context, since no action was taken yet */+ if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */+ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported inputSize, too large (or negative) */+ if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */+ if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */+ assert(acceleration >= 1);++ lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);++ /* Update context state */+ if (dictDirective == usingDictCtx) {+ /* Subsequent linked blocks can't use the dictionary. */+ /* Instead, they use the block we just compressed. */+ cctx->dictCtx = NULL;+ cctx->dictSize = (U32)inputSize;+ } else {+ cctx->dictSize += (U32)inputSize;+ }+ cctx->currentOffset += (U32)inputSize;+ cctx->tableType = (U16)tableType;++ if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */++ /* First Byte */+ LZ4_putPosition(ip, cctx->hashTable, tableType, base);+ ip++; forwardH = LZ4_hashPosition(ip, tableType);++ /* Main Loop */+ for ( ; ; ) {+ const BYTE* match;+ BYTE* token;+ const BYTE* filledIp;++ /* Find a match */+ if (tableType == byPtr) {+ const BYTE* forwardIp = ip;+ int step = 1;+ int searchMatchNb = acceleration << LZ4_skipTrigger;+ do {+ U32 const h = forwardH;+ ip = forwardIp;+ forwardIp += step;+ step = (searchMatchNb++ >> LZ4_skipTrigger);++ if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;+ assert(ip < mflimitPlusOne);++ match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);+ forwardH = LZ4_hashPosition(forwardIp, tableType);+ LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);++ } while ( (match+LZ4_DISTANCE_MAX < ip)+ || (LZ4_read32(match) != LZ4_read32(ip)) );++ } else { /* byU32, byU16 */++ const BYTE* forwardIp = ip;+ int step = 1;+ int searchMatchNb = acceleration << LZ4_skipTrigger;+ do {+ U32 const h = forwardH;+ U32 const current = (U32)(forwardIp - base);+ U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);+ assert(matchIndex <= current);+ assert(forwardIp - base < (ptrdiff_t)(2 GB - 1));+ ip = forwardIp;+ forwardIp += step;+ step = (searchMatchNb++ >> LZ4_skipTrigger);++ if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;+ assert(ip < mflimitPlusOne);++ if (dictDirective == usingDictCtx) {+ if (matchIndex < startIndex) {+ /* there was no match, try the dictionary */+ assert(tableType == byU32);+ matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);+ match = dictBase + matchIndex;+ matchIndex += dictDelta; /* make dictCtx index comparable with current context */+ lowLimit = dictionary;+ } else {+ match = base + matchIndex;+ lowLimit = (const BYTE*)source;+ }+ } else if (dictDirective==usingExtDict) {+ if (matchIndex < startIndex) {+ DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex);+ assert(startIndex - matchIndex >= MINMATCH);+ match = dictBase + matchIndex;+ lowLimit = dictionary;+ } else {+ match = base + matchIndex;+ lowLimit = (const BYTE*)source;+ }+ } else { /* single continuous memory segment */+ match = base + matchIndex;+ }+ forwardH = LZ4_hashPosition(forwardIp, tableType);+ LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);++ DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex);+ if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */+ assert(matchIndex < current);+ if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX))+ && (matchIndex+LZ4_DISTANCE_MAX < current)) {+ continue;+ } /* too far */+ assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */++ if (LZ4_read32(match) == LZ4_read32(ip)) {+ if (maybe_extMem) offset = current - matchIndex;+ break; /* match found */+ }++ } while(1);+ }++ /* Catch up */+ filledIp = ip;+ while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }++ /* Encode Literals */+ { unsigned const litLength = (unsigned)(ip - anchor);+ token = op++;+ if ((outputDirective == limitedOutput) && /* Check output buffer overflow */+ (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) {+ return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */+ }+ if ((outputDirective == fillOutput) &&+ (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) {+ op--;+ goto _last_literals;+ }+ if (litLength >= RUN_MASK) {+ int len = (int)(litLength - RUN_MASK);+ *token = (RUN_MASK<<ML_BITS);+ for(; len >= 255 ; len-=255) *op++ = 255;+ *op++ = (BYTE)len;+ }+ else *token = (BYTE)(litLength<<ML_BITS);++ /* Copy Literals */+ LZ4_wildCopy8(op, anchor, op+litLength);+ op+=litLength;+ DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",+ (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));+ }++_next_match:+ /* at this stage, the following variables must be correctly set :+ * - ip : at start of LZ operation+ * - match : at start of previous pattern occurence; can be within current prefix, or within extDict+ * - offset : if maybe_ext_memSegment==1 (constant)+ * - lowLimit : must be == dictionary to mean "match is within extDict"; must be == source otherwise+ * - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written+ */++ if ((outputDirective == fillOutput) &&+ (op + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit)) {+ /* the match was too close to the end, rewind and go to last literals */+ op = token;+ goto _last_literals;+ }++ /* Encode Offset */+ if (maybe_extMem) { /* static test */+ DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source));+ assert(offset <= LZ4_DISTANCE_MAX && offset > 0);+ LZ4_writeLE16(op, (U16)offset); op+=2;+ } else {+ DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match));+ assert(ip-match <= LZ4_DISTANCE_MAX);+ LZ4_writeLE16(op, (U16)(ip - match)); op+=2;+ }++ /* Encode MatchLength */+ { unsigned matchCode;++ if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx)+ && (lowLimit==dictionary) /* match within extDict */ ) {+ const BYTE* limit = ip + (dictEnd-match);+ assert(dictEnd > match);+ if (limit > matchlimit) limit = matchlimit;+ matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit);+ ip += (size_t)matchCode + MINMATCH;+ if (ip==limit) {+ unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit);+ matchCode += more;+ ip += more;+ }+ DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH);+ } else {+ matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);+ ip += (size_t)matchCode + MINMATCH;+ DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH);+ }++ if ((outputDirective) && /* Check output buffer overflow */+ (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) {+ if (outputDirective == fillOutput) {+ /* Match description too long : reduce it */+ U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255;+ ip -= matchCode - newMatchCode;+ assert(newMatchCode < matchCode);+ matchCode = newMatchCode;+ if (unlikely(ip <= filledIp)) {+ /* We have already filled up to filledIp so if ip ends up less than filledIp+ * we have positions in the hash table beyond the current position. This is+ * a problem if we reuse the hash table. So we have to remove these positions+ * from the hash table.+ */+ const BYTE* ptr;+ DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip));+ for (ptr = ip; ptr <= filledIp; ++ptr) {+ U32 const h = LZ4_hashPosition(ptr, tableType);+ LZ4_clearHash(h, cctx->hashTable, tableType);+ }+ }+ } else {+ assert(outputDirective == limitedOutput);+ return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */+ }+ }+ if (matchCode >= ML_MASK) {+ *token += ML_MASK;+ matchCode -= ML_MASK;+ LZ4_write32(op, 0xFFFFFFFF);+ while (matchCode >= 4*255) {+ op+=4;+ LZ4_write32(op, 0xFFFFFFFF);+ matchCode -= 4*255;+ }+ op += matchCode / 255;+ *op++ = (BYTE)(matchCode % 255);+ } else+ *token += (BYTE)(matchCode);+ }+ /* Ensure we have enough space for the last literals. */+ assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit));++ anchor = ip;++ /* Test end of chunk */+ if (ip >= mflimitPlusOne) break;++ /* Fill table */+ LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);++ /* Test next position */+ if (tableType == byPtr) {++ match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);+ LZ4_putPosition(ip, cctx->hashTable, tableType, base);+ if ( (match+LZ4_DISTANCE_MAX >= ip)+ && (LZ4_read32(match) == LZ4_read32(ip)) )+ { token=op++; *token=0; goto _next_match; }++ } else { /* byU32, byU16 */++ U32 const h = LZ4_hashPosition(ip, tableType);+ U32 const current = (U32)(ip-base);+ U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);+ assert(matchIndex < current);+ if (dictDirective == usingDictCtx) {+ if (matchIndex < startIndex) {+ /* there was no match, try the dictionary */+ matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);+ match = dictBase + matchIndex;+ lowLimit = dictionary; /* required for match length counter */+ matchIndex += dictDelta;+ } else {+ match = base + matchIndex;+ lowLimit = (const BYTE*)source; /* required for match length counter */+ }+ } else if (dictDirective==usingExtDict) {+ if (matchIndex < startIndex) {+ match = dictBase + matchIndex;+ lowLimit = dictionary; /* required for match length counter */+ } else {+ match = base + matchIndex;+ lowLimit = (const BYTE*)source; /* required for match length counter */+ }+ } else { /* single memory segment */+ match = base + matchIndex;+ }+ LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);+ assert(matchIndex < current);+ if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)+ && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current))+ && (LZ4_read32(match) == LZ4_read32(ip)) ) {+ token=op++;+ *token=0;+ if (maybe_extMem) offset = current - matchIndex;+ DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",+ (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));+ goto _next_match;+ }+ }++ /* Prepare next loop */+ forwardH = LZ4_hashPosition(++ip, tableType);++ }++_last_literals:+ /* Encode Last Literals */+ { size_t lastRun = (size_t)(iend - anchor);+ if ( (outputDirective) && /* Check output buffer overflow */+ (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) {+ if (outputDirective == fillOutput) {+ /* adapt lastRun to fill 'dst' */+ assert(olimit >= op);+ lastRun = (size_t)(olimit-op) - 1;+ lastRun -= (lastRun+240)/255;+ } else {+ assert(outputDirective == limitedOutput);+ return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */+ }+ }+ if (lastRun >= RUN_MASK) {+ size_t accumulator = lastRun - RUN_MASK;+ *op++ = RUN_MASK << ML_BITS;+ for(; accumulator >= 255 ; accumulator-=255) *op++ = 255;+ *op++ = (BYTE) accumulator;+ } else {+ *op++ = (BYTE)(lastRun<<ML_BITS);+ }+ memcpy(op, anchor, lastRun);+ ip = anchor + lastRun;+ op += lastRun;+ }++ if (outputDirective == fillOutput) {+ *inputConsumed = (int) (((const char*)ip)-source);+ }+ DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, (int)(((char*)op) - dest));+ result = (int)(((char*)op) - dest);+ assert(result > 0);+ return result;+}+++int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)+{+ LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse;+ assert(ctx != NULL);+ if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;+ if (maxOutputSize >= LZ4_compressBound(inputSize)) {+ if (inputSize < LZ4_64Klimit) {+ return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration);+ } else {+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32;+ return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);+ }+ } else {+ if (inputSize < LZ4_64Klimit) {+ return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);+ } else {+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32;+ return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);+ }+ }+}++/**+ * LZ4_compress_fast_extState_fastReset() :+ * A variant of LZ4_compress_fast_extState().+ *+ * Using this variant avoids an expensive initialization step. It is only safe+ * to call if the state buffer is known to be correctly initialized already+ * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of+ * "correctly initialized").+ */+int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration)+{+ LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;+ if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;++ if (dstCapacity >= LZ4_compressBound(srcSize)) {+ if (srcSize < LZ4_64Klimit) {+ const tableType_t tableType = byU16;+ LZ4_prepareTable(ctx, srcSize, tableType);+ if (ctx->currentOffset) {+ return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration);+ } else {+ return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);+ }+ } else {+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32;+ LZ4_prepareTable(ctx, srcSize, tableType);+ return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);+ }+ } else {+ if (srcSize < LZ4_64Klimit) {+ const tableType_t tableType = byU16;+ LZ4_prepareTable(ctx, srcSize, tableType);+ if (ctx->currentOffset) {+ return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration);+ } else {+ return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);+ }+ } else {+ const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32;+ LZ4_prepareTable(ctx, srcSize, tableType);+ return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);+ }+ }+}+++int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)+{+ int result;+#if (LZ4_HEAPMODE)+ LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */+ if (ctxPtr == NULL) return 0;+#else+ LZ4_stream_t ctx;+ LZ4_stream_t* const ctxPtr = &ctx;+#endif+ result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);++#if (LZ4_HEAPMODE)+ FREEMEM(ctxPtr);+#endif+ return result;+}+++int LZ4_compress_default(const char* src, char* dst, int srcSize, int maxOutputSize)+{+ return LZ4_compress_fast(src, dst, srcSize, maxOutputSize, 1);+}+++/* hidden debug function */+/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */+int LZ4_compress_fast_force(const char* src, char* dst, int srcSize, int dstCapacity, int acceleration)+{+ LZ4_stream_t ctx;+ LZ4_initStream(&ctx, sizeof(ctx));++ if (srcSize < LZ4_64Klimit) {+ return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, byU16, noDict, noDictIssue, acceleration);+ } else {+ tableType_t const addrMode = (sizeof(void*) > 4) ? byU32 : byPtr;+ return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, addrMode, noDict, noDictIssue, acceleration);+ }+}+++/* Note!: This function leaves the stream in an unclean/broken state!+ * It is not safe to subsequently use the same state with a _fastReset() or+ * _continue() call without resetting it. */+static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize)+{+ void* const s = LZ4_initStream(state, sizeof (*state));+ assert(s != NULL); (void)s;++ if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */+ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);+ } else {+ if (*srcSizePtr < LZ4_64Klimit) {+ return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1);+ } else {+ tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32;+ return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, 1);+ } }+}+++int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)+{+#if (LZ4_HEAPMODE)+ LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */+ if (ctx == NULL) return 0;+#else+ LZ4_stream_t ctxBody;+ LZ4_stream_t* ctx = &ctxBody;+#endif++ int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize);++#if (LZ4_HEAPMODE)+ FREEMEM(ctx);+#endif+ return result;+}++++/*-******************************+* Streaming functions+********************************/++LZ4_stream_t* LZ4_createStream(void)+{+ LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));+ LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */+ DEBUGLOG(4, "LZ4_createStream %p", lz4s);+ if (lz4s == NULL) return NULL;+ LZ4_initStream(lz4s, sizeof(*lz4s));+ return lz4s;+}++#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :+ it reports an aligment of 8-bytes,+ while actually aligning LZ4_stream_t on 4 bytes. */+static size_t LZ4_stream_t_alignment(void)+{+ struct { char c; LZ4_stream_t t; } t_a;+ return sizeof(t_a) - sizeof(t_a.t);+}+#endif++LZ4_stream_t* LZ4_initStream (void* buffer, size_t size)+{+ DEBUGLOG(5, "LZ4_initStream");+ if (buffer == NULL) { return NULL; }+ if (size < sizeof(LZ4_stream_t)) { return NULL; }+#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :+ it reports an aligment of 8-bytes,+ while actually aligning LZ4_stream_t on 4 bytes. */+ if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) { return NULL; } /* alignment check */+#endif+ MEM_INIT(buffer, 0, sizeof(LZ4_stream_t));+ return (LZ4_stream_t*)buffer;+}++/* resetStream is now deprecated,+ * prefer initStream() which is more general */+void LZ4_resetStream (LZ4_stream_t* LZ4_stream)+{+ DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream);+ MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));+}++void LZ4_resetStream_fast(LZ4_stream_t* ctx) {+ LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);+}++int LZ4_freeStream (LZ4_stream_t* LZ4_stream)+{+ if (!LZ4_stream) return 0; /* support free on NULL */+ DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream);+ FREEMEM(LZ4_stream);+ return (0);+}+++#define HASH_UNIT sizeof(reg_t)+int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)+{+ LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse;+ const tableType_t tableType = byU32;+ const BYTE* p = (const BYTE*)dictionary;+ const BYTE* const dictEnd = p + dictSize;+ const BYTE* base;++ DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict);++ /* It's necessary to reset the context,+ * and not just continue it with prepareTable()+ * to avoid any risk of generating overflowing matchIndex+ * when compressing using this dictionary */+ LZ4_resetStream(LZ4_dict);++ /* We always increment the offset by 64 KB, since, if the dict is longer,+ * we truncate it to the last 64k, and if it's shorter, we still want to+ * advance by a whole window length so we can provide the guarantee that+ * there are only valid offsets in the window, which allows an optimization+ * in LZ4_compress_fast_continue() where it uses noDictIssue even when the+ * dictionary isn't a full 64k. */+ dict->currentOffset += 64 KB;++ if (dictSize < (int)HASH_UNIT) {+ return 0;+ }++ if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;+ base = dictEnd - dict->currentOffset;+ dict->dictionary = p;+ dict->dictSize = (U32)(dictEnd - p);+ dict->tableType = tableType;++ while (p <= dictEnd-HASH_UNIT) {+ LZ4_putPosition(p, dict->hashTable, tableType, base);+ p+=3;+ }++ return (int)dict->dictSize;+}++void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) {+ const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL :+ &(dictionaryStream->internal_donotuse);++ DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)",+ workingStream, dictionaryStream,+ dictCtx != NULL ? dictCtx->dictSize : 0);++ /* Calling LZ4_resetStream_fast() here makes sure that changes will not be+ * erased by subsequent calls to LZ4_resetStream_fast() in case stream was+ * marked as having dirty context, e.g. requiring full reset.+ */+ LZ4_resetStream_fast(workingStream);++ if (dictCtx != NULL) {+ /* If the current offset is zero, we will never look in the+ * external dictionary context, since there is no value a table+ * entry can take that indicate a miss. In that case, we need+ * to bump the offset to something non-zero.+ */+ if (workingStream->internal_donotuse.currentOffset == 0) {+ workingStream->internal_donotuse.currentOffset = 64 KB;+ }++ /* Don't actually attach an empty dictionary.+ */+ if (dictCtx->dictSize == 0) {+ dictCtx = NULL;+ }+ }+ workingStream->internal_donotuse.dictCtx = dictCtx;+}+++static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize)+{+ assert(nextSize >= 0);+ if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */+ /* rescale hash table */+ U32 const delta = LZ4_dict->currentOffset - 64 KB;+ const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;+ int i;+ DEBUGLOG(4, "LZ4_renormDictT");+ for (i=0; i<LZ4_HASH_SIZE_U32; i++) {+ if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0;+ else LZ4_dict->hashTable[i] -= delta;+ }+ LZ4_dict->currentOffset = 64 KB;+ if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB;+ LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize;+ }+}+++int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream,+ const char* source, char* dest,+ int inputSize, int maxOutputSize,+ int acceleration)+{+ const tableType_t tableType = byU32;+ LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;+ const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize;++ DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize);++ if (streamPtr->dirty) { return 0; } /* Uninitialized structure detected */+ LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */+ if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;++ /* invalidate tiny dictionaries */+ if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */+ && (dictEnd != (const BYTE*)source) ) {+ DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary);+ streamPtr->dictSize = 0;+ streamPtr->dictionary = (const BYTE*)source;+ dictEnd = (const BYTE*)source;+ }++ /* Check overlapping input/dictionary space */+ { const BYTE* sourceEnd = (const BYTE*) source + inputSize;+ if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) {+ streamPtr->dictSize = (U32)(dictEnd - sourceEnd);+ if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB;+ if (streamPtr->dictSize < 4) streamPtr->dictSize = 0;+ streamPtr->dictionary = dictEnd - streamPtr->dictSize;+ }+ }++ /* prefix mode : source data follows dictionary */+ if (dictEnd == (const BYTE*)source) {+ if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))+ return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);+ else+ return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration);+ }++ /* external dictionary mode */+ { int result;+ if (streamPtr->dictCtx) {+ /* We depend here on the fact that dictCtx'es (produced by+ * LZ4_loadDict) guarantee that their tables contain no references+ * to offsets between dictCtx->currentOffset - 64 KB and+ * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe+ * to use noDictIssue even when the dict isn't a full 64 KB.+ */+ if (inputSize > 4 KB) {+ /* For compressing large blobs, it is faster to pay the setup+ * cost to copy the dictionary's tables into the active context,+ * so that the compression loop is only looking into one table.+ */+ memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));+ result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);+ } else {+ result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);+ }+ } else {+ if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {+ result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);+ } else {+ result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);+ }+ }+ streamPtr->dictionary = (const BYTE*)source;+ streamPtr->dictSize = (U32)inputSize;+ return result;+ }+}+++/* Hidden debug function, to force-test external dictionary mode */+int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize)+{+ LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse;+ int result;++ LZ4_renormDictT(streamPtr, srcSize);++ if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {+ result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1);+ } else {+ result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);+ }++ streamPtr->dictionary = (const BYTE*)source;+ streamPtr->dictSize = (U32)srcSize;++ return result;+}+++/*! LZ4_saveDict() :+ * If previously compressed data block is not guaranteed to remain available at its memory location,+ * save it into a safer place (char* safeBuffer).+ * Note : you don't need to call LZ4_loadDict() afterwards,+ * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().+ * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.+ */+int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)+{+ LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;+ const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;++ if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */+ if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }++ memmove(safeBuffer, previousDictEnd - dictSize, dictSize);++ dict->dictionary = (const BYTE*)safeBuffer;+ dict->dictSize = (U32)dictSize;++ return dictSize;+}++++/*-*******************************+ * Decompression functions+ ********************************/++typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;+typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive;++#undef MIN+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )++/* Read the variable-length literal or match length.+ *+ * ip - pointer to use as input.+ * lencheck - end ip. Return an error if ip advances >= lencheck.+ * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so.+ * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so.+ * error (output) - error code. Should be set to 0 before call.+ */+typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error;+LZ4_FORCE_INLINE unsigned+read_variable_length(const BYTE**ip, const BYTE* lencheck, int loop_check, int initial_check, variable_length_error* error)+{+ unsigned length = 0;+ unsigned s;+ if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */+ *error = initial_error;+ return length;+ }+ do {+ s = **ip;+ (*ip)++;+ length += s;+ if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */+ *error = loop_error;+ return length;+ }+ } while (s==255);++ return length;+}++/*! LZ4_decompress_generic() :+ * This generic decompression function covers all use cases.+ * It shall be instantiated several times, using different sets of directives.+ * Note that it is important for performance that this function really get inlined,+ * in order to remove useless branches during compilation optimization.+ */+LZ4_FORCE_INLINE int+LZ4_decompress_generic(+ const char* const src,+ char* const dst,+ int srcSize,+ int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */++ endCondition_directive endOnInput, /* endOnOutputSize, endOnInputSize */+ earlyEnd_directive partialDecoding, /* full, partial */+ dict_directive dict, /* noDict, withPrefix64k, usingExtDict */+ const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */+ const BYTE* const dictStart, /* only if dict==usingExtDict */+ const size_t dictSize /* note : = 0 if noDict */+ )+{+ if (src == NULL) { return -1; }++ { const BYTE* ip = (const BYTE*) src;+ const BYTE* const iend = ip + srcSize;++ BYTE* op = (BYTE*) dst;+ BYTE* const oend = op + outputSize;+ BYTE* cpy;++ const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize;++ const int safeDecode = (endOnInput==endOnInputSize);+ const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));+++ /* Set up the "end" pointers for the shortcut. */+ const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/;+ const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/;++ const BYTE* match;+ size_t offset;+ unsigned token;+ size_t length;+++ DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize);++ /* Special cases */+ assert(lowPrefix <= op);+ if ((endOnInput) && (unlikely(outputSize==0))) {+ /* Empty output buffer */+ if (partialDecoding) return 0;+ return ((srcSize==1) && (*ip==0)) ? 0 : -1;+ }+ if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); }+ if ((endOnInput) && unlikely(srcSize==0)) { return -1; }++ /* Currently the fast loop shows a regression on qualcomm arm chips. */+#if LZ4_FAST_DEC_LOOP+ if ((oend - op) < FASTLOOP_SAFE_DISTANCE) {+ DEBUGLOG(6, "skip fast decode loop");+ goto safe_decode;+ }++ /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */+ while (1) {+ /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */+ assert(oend - op >= FASTLOOP_SAFE_DISTANCE);+ if (endOnInput) { assert(ip < iend); }+ token = *ip++;+ length = token >> ML_BITS; /* literal length */++ assert(!endOnInput || ip <= iend); /* ip < iend before the increment */++ /* decode literal length */+ if (length == RUN_MASK) {+ variable_length_error error = ok;+ length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error);+ if (error == initial_error) { goto _output_error; }+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */+ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */++ /* copy literals */+ cpy = op+length;+ LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);+ if (endOnInput) { /* LZ4_decompress_safe() */+ if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; }+ LZ4_wildCopy32(op, ip, cpy);+ } else { /* LZ4_decompress_fast() */+ if (cpy>oend-8) { goto safe_literal_copy; }+ LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :+ * it doesn't know input length, and only relies on end-of-block properties */+ }+ ip += length; op = cpy;+ } else {+ cpy = op+length;+ if (endOnInput) { /* LZ4_decompress_safe() */+ DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length);+ /* We don't need to check oend, since we check it once for each loop below */+ if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; }+ /* Literals can only be 14, but hope compilers optimize if we copy by a register size */+ memcpy(op, ip, 16);+ } else { /* LZ4_decompress_fast() */+ /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :+ * it doesn't know input length, and relies on end-of-block properties */+ memcpy(op, ip, 8);+ if (length > 8) { memcpy(op+8, ip+8, 8); }+ }+ ip += length; op = cpy;+ }++ /* get offset */+ offset = LZ4_readLE16(ip); ip+=2;+ match = op - offset;+ assert(match <= op);++ /* get matchlength */+ length = token & ML_MASK;++ if (length == ML_MASK) {+ variable_length_error error = ok;+ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */+ length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error);+ if (error != ok) { goto _output_error; }+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */+ length += MINMATCH;+ if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {+ goto safe_match_copy;+ }+ } else {+ length += MINMATCH;+ if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {+ goto safe_match_copy;+ }++ /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */+ if ((dict == withPrefix64k) || (match >= lowPrefix)) {+ if (offset >= 8) {+ assert(match >= lowPrefix);+ assert(match <= op);+ assert(op + 18 <= oend);++ memcpy(op, match, 8);+ memcpy(op+8, match+8, 8);+ memcpy(op+16, match+16, 2);+ op += length;+ continue;+ } } }++ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */+ /* match starting within external dictionary */+ if ((dict==usingExtDict) && (match < lowPrefix)) {+ if (unlikely(op+length > oend-LASTLITERALS)) {+ if (partialDecoding) {+ length = MIN(length, (size_t)(oend-op)); /* reach end of buffer */+ } else {+ goto _output_error; /* end-of-block condition violated */+ } }++ if (length <= (size_t)(lowPrefix-match)) {+ /* match fits entirely within external dictionary : just copy */+ memmove(op, dictEnd - (lowPrefix-match), length);+ op += length;+ } else {+ /* match stretches into both external dictionary and current block */+ size_t const copySize = (size_t)(lowPrefix - match);+ size_t const restSize = length - copySize;+ memcpy(op, dictEnd - copySize, copySize);+ op += copySize;+ if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */+ BYTE* const endOfMatch = op + restSize;+ const BYTE* copyFrom = lowPrefix;+ while (op < endOfMatch) { *op++ = *copyFrom++; }+ } else {+ memcpy(op, lowPrefix, restSize);+ op += restSize;+ } }+ continue;+ }++ /* copy match within block */+ cpy = op + length;++ assert((op <= oend) && (oend-op >= 32));+ if (unlikely(offset<16)) {+ LZ4_memcpy_using_offset(op, match, cpy, offset);+ } else {+ LZ4_wildCopy32(op, match, cpy);+ }++ op = cpy; /* wildcopy correction */+ }+ safe_decode:+#endif++ /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */+ while (1) {+ token = *ip++;+ length = token >> ML_BITS; /* literal length */++ assert(!endOnInput || ip <= iend); /* ip < iend before the increment */++ /* A two-stage shortcut for the most common case:+ * 1) If the literal length is 0..14, and there is enough space,+ * enter the shortcut and copy 16 bytes on behalf of the literals+ * (in the fast mode, only 8 bytes can be safely copied this way).+ * 2) Further if the match length is 4..18, copy 18 bytes in a similar+ * manner; but we ensure that there's enough space in the output for+ * those 18 bytes earlier, upon entering the shortcut (in other words,+ * there is a combined check for both stages).+ */+ if ( (endOnInput ? length != RUN_MASK : length <= 8)+ /* strictly "less than" on input, to re-enter the loop with at least one byte */+ && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) {+ /* Copy the literals */+ memcpy(op, ip, endOnInput ? 16 : 8);+ op += length; ip += length;++ /* The second stage: prepare for match copying, decode full info.+ * If it doesn't work out, the info won't be wasted. */+ length = token & ML_MASK; /* match length */+ offset = LZ4_readLE16(ip); ip += 2;+ match = op - offset;+ assert(match <= op); /* check overflow */++ /* Do not deal with overlapping matches. */+ if ( (length != ML_MASK)+ && (offset >= 8)+ && (dict==withPrefix64k || match >= lowPrefix) ) {+ /* Copy the match. */+ memcpy(op + 0, match + 0, 8);+ memcpy(op + 8, match + 8, 8);+ memcpy(op +16, match +16, 2);+ op += length + MINMATCH;+ /* Both stages worked, load the next token. */+ continue;+ }++ /* The second stage didn't work out, but the info is ready.+ * Propel it right to the point of match copying. */+ goto _copy_match;+ }++ /* decode literal length */+ if (length == RUN_MASK) {+ variable_length_error error = ok;+ length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error);+ if (error == initial_error) { goto _output_error; }+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */+ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */+ }++ /* copy literals */+ cpy = op+length;+#if LZ4_FAST_DEC_LOOP+ safe_literal_copy:+#endif+ LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);+ if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) )+ || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )+ {+ /* We've either hit the input parsing restriction or the output parsing restriction.+ * If we've hit the input parsing condition then this must be the last sequence.+ * If we've hit the output parsing condition then we are either using partialDecoding+ * or we've hit the output parsing condition.+ */+ if (partialDecoding) {+ /* Since we are partial decoding we may be in this block because of the output parsing+ * restriction, which is not valid since the output buffer is allowed to be undersized.+ */+ assert(endOnInput);+ /* If we're in this block because of the input parsing condition, then we must be on the+ * last sequence (or invalid), so we must check that we exactly consume the input.+ */+ if ((ip+length>iend-(2+1+LASTLITERALS)) && (ip+length != iend)) { goto _output_error; }+ assert(ip+length <= iend);+ /* We are finishing in the middle of a literals segment.+ * Break after the copy.+ */+ if (cpy > oend) {+ cpy = oend;+ assert(op<=oend);+ length = (size_t)(oend-op);+ }+ assert(ip+length <= iend);+ } else {+ /* We must be on the last sequence because of the parsing limitations so check+ * that we exactly regenerate the original size (must be exact when !endOnInput).+ */+ if ((!endOnInput) && (cpy != oend)) { goto _output_error; }+ /* We must be on the last sequence (or invalid) because of the parsing limitations+ * so check that we exactly consume the input and don't overrun the output buffer.+ */+ if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; }+ }+ memmove(op, ip, length); /* supports overlapping memory regions, which only matters for in-place decompression scenarios */+ ip += length;+ op += length;+ /* Necessarily EOF when !partialDecoding. When partialDecoding+ * it is EOF if we've either filled the output buffer or hit+ * the input parsing restriction.+ */+ if (!partialDecoding || (cpy == oend) || (ip == iend)) {+ break;+ }+ } else {+ LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */+ ip += length; op = cpy;+ }++ /* get offset */+ offset = LZ4_readLE16(ip); ip+=2;+ match = op - offset;++ /* get matchlength */+ length = token & ML_MASK;++ _copy_match:+ if (length == ML_MASK) {+ variable_length_error error = ok;+ length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error);+ if (error != ok) goto _output_error;+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */+ }+ length += MINMATCH;++#if LZ4_FAST_DEC_LOOP+ safe_match_copy:+#endif+ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */+ /* match starting within external dictionary */+ if ((dict==usingExtDict) && (match < lowPrefix)) {+ if (unlikely(op+length > oend-LASTLITERALS)) {+ if (partialDecoding) length = MIN(length, (size_t)(oend-op));+ else goto _output_error; /* doesn't respect parsing restriction */+ }++ if (length <= (size_t)(lowPrefix-match)) {+ /* match fits entirely within external dictionary : just copy */+ memmove(op, dictEnd - (lowPrefix-match), length);+ op += length;+ } else {+ /* match stretches into both external dictionary and current block */+ size_t const copySize = (size_t)(lowPrefix - match);+ size_t const restSize = length - copySize;+ memcpy(op, dictEnd - copySize, copySize);+ op += copySize;+ if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */+ BYTE* const endOfMatch = op + restSize;+ const BYTE* copyFrom = lowPrefix;+ while (op < endOfMatch) *op++ = *copyFrom++;+ } else {+ memcpy(op, lowPrefix, restSize);+ op += restSize;+ } }+ continue;+ }+ assert(match >= lowPrefix);++ /* copy match within block */+ cpy = op + length;++ /* partialDecoding : may end anywhere within the block */+ assert(op<=oend);+ if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {+ size_t const mlen = MIN(length, (size_t)(oend-op));+ const BYTE* const matchEnd = match + mlen;+ BYTE* const copyEnd = op + mlen;+ if (matchEnd > op) { /* overlap copy */+ while (op < copyEnd) { *op++ = *match++; }+ } else {+ memcpy(op, match, mlen);+ }+ op = copyEnd;+ if (op == oend) { break; }+ continue;+ }++ if (unlikely(offset<8)) {+ LZ4_write32(op, 0); /* silence msan warning when offset==0 */+ op[0] = match[0];+ op[1] = match[1];+ op[2] = match[2];+ op[3] = match[3];+ match += inc32table[offset];+ memcpy(op+4, match, 4);+ match -= dec64table[offset];+ } else {+ memcpy(op, match, 8);+ match += 8;+ }+ op += 8;++ if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {+ BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1);+ if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */+ if (op < oCopyLimit) {+ LZ4_wildCopy8(op, match, oCopyLimit);+ match += oCopyLimit - op;+ op = oCopyLimit;+ }+ while (op < cpy) { *op++ = *match++; }+ } else {+ memcpy(op, match, 8);+ if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); }+ }+ op = cpy; /* wildcopy correction */+ }++ /* end of decoding */+ if (endOnInput) {+ return (int) (((char*)op)-dst); /* Nb of output bytes decoded */+ } else {+ return (int) (((const char*)ip)-src); /* Nb of input bytes read */+ }++ /* Overflow error detected */+ _output_error:+ return (int) (-(((const char*)ip)-src))-1;+ }+}+++/*===== Instantiate the API decoding functions. =====*/++LZ4_FORCE_O2_GCC_PPC64LE+int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)+{+ return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,+ endOnInputSize, decode_full_block, noDict,+ (BYTE*)dest, NULL, 0);+}++LZ4_FORCE_O2_GCC_PPC64LE+int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity)+{+ dstCapacity = MIN(targetOutputSize, dstCapacity);+ return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity,+ endOnInputSize, partial_decode,+ noDict, (BYTE*)dst, NULL, 0);+}++LZ4_FORCE_O2_GCC_PPC64LE+int LZ4_decompress_fast(const char* source, char* dest, int originalSize)+{+ return LZ4_decompress_generic(source, dest, 0, originalSize,+ endOnOutputSize, decode_full_block, withPrefix64k,+ (BYTE*)dest - 64 KB, NULL, 0);+}++/*===== Instantiate a few more decoding cases, used more than once. =====*/++LZ4_FORCE_O2_GCC_PPC64LE /* Exported, an obsolete API function. */+int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)+{+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,+ endOnInputSize, decode_full_block, withPrefix64k,+ (BYTE*)dest - 64 KB, NULL, 0);+}++/* Another obsolete API function, paired with the previous one. */+int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)+{+ /* LZ4_decompress_fast doesn't validate match offsets,+ * and thus serves well with any prefixed dictionary. */+ return LZ4_decompress_fast(source, dest, originalSize);+}++LZ4_FORCE_O2_GCC_PPC64LE+static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize,+ size_t prefixSize)+{+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,+ endOnInputSize, decode_full_block, noDict,+ (BYTE*)dest-prefixSize, NULL, 0);+}++LZ4_FORCE_O2_GCC_PPC64LE+int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,+ int compressedSize, int maxOutputSize,+ const void* dictStart, size_t dictSize)+{+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,+ endOnInputSize, decode_full_block, usingExtDict,+ (BYTE*)dest, (const BYTE*)dictStart, dictSize);+}++LZ4_FORCE_O2_GCC_PPC64LE+static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize,+ const void* dictStart, size_t dictSize)+{+ return LZ4_decompress_generic(source, dest, 0, originalSize,+ endOnOutputSize, decode_full_block, usingExtDict,+ (BYTE*)dest, (const BYTE*)dictStart, dictSize);+}++/* The "double dictionary" mode, for use with e.g. ring buffers: the first part+ * of the dictionary is passed as prefix, and the second via dictStart + dictSize.+ * These routines are used only once, in LZ4_decompress_*_continue().+ */+LZ4_FORCE_INLINE+int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize,+ size_t prefixSize, const void* dictStart, size_t dictSize)+{+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,+ endOnInputSize, decode_full_block, usingExtDict,+ (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);+}++LZ4_FORCE_INLINE+int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize,+ size_t prefixSize, const void* dictStart, size_t dictSize)+{+ return LZ4_decompress_generic(source, dest, 0, originalSize,+ endOnOutputSize, decode_full_block, usingExtDict,+ (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);+}++/*===== streaming decompression functions =====*/++LZ4_streamDecode_t* LZ4_createStreamDecode(void)+{+ LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t));+ LZ4_STATIC_ASSERT(LZ4_STREAMDECODESIZE >= sizeof(LZ4_streamDecode_t_internal)); /* A compilation error here means LZ4_STREAMDECODESIZE is not large enough */+ return lz4s;+}++int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)+{+ if (LZ4_stream == NULL) { return 0; } /* support free on NULL */+ FREEMEM(LZ4_stream);+ return 0;+}++/*! LZ4_setStreamDecode() :+ * Use this function to instruct where to find the dictionary.+ * This function is not necessary if previous data is still available where it was decoded.+ * Loading a size of 0 is allowed (same effect as no dictionary).+ * @return : 1 if OK, 0 if error+ */+int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize)+{+ LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;+ lz4sd->prefixSize = (size_t) dictSize;+ lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize;+ lz4sd->externalDict = NULL;+ lz4sd->extDictSize = 0;+ return 1;+}++/*! LZ4_decoderRingBufferSize() :+ * when setting a ring buffer for streaming decompression (optional scenario),+ * provides the minimum size of this ring buffer+ * to be compatible with any source respecting maxBlockSize condition.+ * Note : in a ring buffer scenario,+ * blocks are presumed decompressed next to each other.+ * When not enough space remains for next block (remainingSize < maxBlockSize),+ * decoding resumes from beginning of ring buffer.+ * @return : minimum ring buffer size,+ * or 0 if there is an error (invalid maxBlockSize).+ */+int LZ4_decoderRingBufferSize(int maxBlockSize)+{+ if (maxBlockSize < 0) return 0;+ if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0;+ if (maxBlockSize < 16) maxBlockSize = 16;+ return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize);+}++/*+*_continue() :+ These decoding functions allow decompression of multiple blocks in "streaming" mode.+ Previously decoded blocks must still be available at the memory position where they were decoded.+ If it's not possible, save the relevant part of decoded data into a safe buffer,+ and indicate where it stands using LZ4_setStreamDecode()+*/+LZ4_FORCE_O2_GCC_PPC64LE+int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)+{+ LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;+ int result;++ if (lz4sd->prefixSize == 0) {+ /* The first call, no dictionary yet. */+ assert(lz4sd->extDictSize == 0);+ result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);+ if (result <= 0) return result;+ lz4sd->prefixSize = (size_t)result;+ lz4sd->prefixEnd = (BYTE*)dest + result;+ } else if (lz4sd->prefixEnd == (BYTE*)dest) {+ /* They're rolling the current segment. */+ if (lz4sd->prefixSize >= 64 KB - 1)+ result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);+ else if (lz4sd->extDictSize == 0)+ result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize,+ lz4sd->prefixSize);+ else+ result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize,+ lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);+ if (result <= 0) return result;+ lz4sd->prefixSize += (size_t)result;+ lz4sd->prefixEnd += result;+ } else {+ /* The buffer wraps around, or they're switching to another buffer. */+ lz4sd->extDictSize = lz4sd->prefixSize;+ lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;+ result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize,+ lz4sd->externalDict, lz4sd->extDictSize);+ if (result <= 0) return result;+ lz4sd->prefixSize = (size_t)result;+ lz4sd->prefixEnd = (BYTE*)dest + result;+ }++ return result;+}++LZ4_FORCE_O2_GCC_PPC64LE+int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)+{+ LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;+ int result;+ assert(originalSize >= 0);++ if (lz4sd->prefixSize == 0) {+ assert(lz4sd->extDictSize == 0);+ result = LZ4_decompress_fast(source, dest, originalSize);+ if (result <= 0) return result;+ lz4sd->prefixSize = (size_t)originalSize;+ lz4sd->prefixEnd = (BYTE*)dest + originalSize;+ } else if (lz4sd->prefixEnd == (BYTE*)dest) {+ if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0)+ result = LZ4_decompress_fast(source, dest, originalSize);+ else+ result = LZ4_decompress_fast_doubleDict(source, dest, originalSize,+ lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);+ if (result <= 0) return result;+ lz4sd->prefixSize += (size_t)originalSize;+ lz4sd->prefixEnd += originalSize;+ } else {+ lz4sd->extDictSize = lz4sd->prefixSize;+ lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;+ result = LZ4_decompress_fast_extDict(source, dest, originalSize,+ lz4sd->externalDict, lz4sd->extDictSize);+ if (result <= 0) return result;+ lz4sd->prefixSize = (size_t)originalSize;+ lz4sd->prefixEnd = (BYTE*)dest + originalSize;+ }++ return result;+}+++/*+Advanced decoding functions :+*_usingDict() :+ These decoding functions work the same as "_continue" ones,+ the dictionary must be explicitly provided within parameters+*/++int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)+{+ if (dictSize==0)+ return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);+ if (dictStart+dictSize == dest) {+ if (dictSize >= 64 KB - 1) {+ return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);+ }+ assert(dictSize >= 0);+ return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize);+ }+ assert(dictSize >= 0);+ return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize);+}++int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)+{+ if (dictSize==0 || dictStart+dictSize == dest)+ return LZ4_decompress_fast(source, dest, originalSize);+ assert(dictSize >= 0);+ return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize);+}+++/*=*************************************************+* Obsolete Functions+***************************************************/+/* obsolete compression functions */+int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)+{+ return LZ4_compress_default(source, dest, inputSize, maxOutputSize);+}+int LZ4_compress(const char* src, char* dest, int srcSize)+{+ return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize));+}+int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize)+{+ return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1);+}+int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize)+{+ return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1);+}+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity)+{+ return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1);+}+int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize)+{+ return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1);+}++/*+These decompression functions are deprecated and should no longer be used.+They are only provided here for compatibility with older user programs.+- LZ4_uncompress is totally equivalent to LZ4_decompress_fast+- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe+*/+int LZ4_uncompress (const char* source, char* dest, int outputSize)+{+ return LZ4_decompress_fast(source, dest, outputSize);+}+int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize)+{+ return LZ4_decompress_safe(source, dest, isize, maxOutputSize);+}++/* Obsolete Streaming functions */++int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }++int LZ4_resetStreamState(void* state, char* inputBuffer)+{+ (void)inputBuffer;+ LZ4_resetStream((LZ4_stream_t*)state);+ return 0;+}++void* LZ4_create (char* inputBuffer)+{+ (void)inputBuffer;+ return LZ4_createStream();+}++char* LZ4_slideInputBuffer (void* state)+{+ /* avoid const char * -> char * conversion warning */+ return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary;+}++#endif /* LZ4_COMMONDEFS_ONLY */
+ lz4/lib/lz4.h view
@@ -0,0 +1,764 @@+/*+ * LZ4 - Fast LZ compression algorithm+ * Header File+ * Copyright (C) 2011-present, Yann Collet.++ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)++ Redistribution and use in source and binary forms, with or without+ modification, are permitted provided that the following conditions are+ met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.+ * Redistributions in binary form must reproduce the above+ copyright notice, this list of conditions and the following disclaimer+ in the documentation and/or other materials provided with the+ distribution.++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.++ You can contact the author at :+ - LZ4 homepage : http://www.lz4.org+ - LZ4 source repository : https://github.com/lz4/lz4+*/+#if defined (__cplusplus)+extern "C" {+#endif++#ifndef LZ4_H_2983827168210+#define LZ4_H_2983827168210++/* --- Dependency --- */+#include <stddef.h> /* size_t */+++/**+ Introduction++ LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core,+ scalable with multi-cores CPU. It features an extremely fast decoder, with speed in+ multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.++ The LZ4 compression library provides in-memory compression and decompression functions.+ It gives full buffer control to user.+ Compression can be done in:+ - a single step (described as Simple Functions)+ - a single step, reusing a context (described in Advanced Functions)+ - unbounded multiple steps (described as Streaming compression)++ lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md).+ Decompressing such a compressed block requires additional metadata.+ Exact metadata depends on exact decompression function.+ For the typical case of LZ4_decompress_safe(),+ metadata includes block's compressed size, and maximum bound of decompressed size.+ Each application is free to encode and pass such metadata in whichever way it wants.++ lz4.h only handle blocks, it can not generate Frames.++ Blocks are different from Frames (doc/lz4_Frame_format.md).+ Frames bundle both blocks and metadata in a specified manner.+ Embedding metadata is required for compressed data to be self-contained and portable.+ Frame format is delivered through a companion API, declared in lz4frame.h.+ The `lz4` CLI can only manage frames.+*/++/*^***************************************************************+* Export parameters+*****************************************************************/+/*+* LZ4_DLL_EXPORT :+* Enable exporting of functions when building a Windows DLL+* LZ4LIB_VISIBILITY :+* Control library symbols visibility.+*/+#ifndef LZ4LIB_VISIBILITY+# if defined(__GNUC__) && (__GNUC__ >= 4)+# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default")))+# else+# define LZ4LIB_VISIBILITY+# endif+#endif+#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)+# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY+#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)+# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/+#else+# define LZ4LIB_API LZ4LIB_VISIBILITY+#endif++/*------ Version ------*/+#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */+#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */+#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */++#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)++#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE+#define LZ4_QUOTE(str) #str+#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)+#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION)++LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */+LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */+++/*-************************************+* Tuning parameter+**************************************/+/*!+ * LZ4_MEMORY_USAGE :+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)+ * Increasing memory usage improves compression ratio.+ * Reduced memory usage may improve speed, thanks to better cache locality.+ * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache+ */+#ifndef LZ4_MEMORY_USAGE+# define LZ4_MEMORY_USAGE 14+#endif+++/*-************************************+* Simple Functions+**************************************/+/*! LZ4_compress_default() :+ * Compresses 'srcSize' bytes from buffer 'src'+ * into already allocated 'dst' buffer of size 'dstCapacity'.+ * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).+ * It also runs faster, so it's a recommended setting.+ * If the function cannot compress 'src' into a more limited 'dst' budget,+ * compression stops *immediately*, and the function result is zero.+ * In which case, 'dst' content is undefined (invalid).+ * srcSize : max supported value is LZ4_MAX_INPUT_SIZE.+ * dstCapacity : size of buffer 'dst' (which must be already allocated)+ * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)+ * or 0 if compression fails+ * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).+ */+LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);++/*! LZ4_decompress_safe() :+ * compressedSize : is the exact complete size of the compressed block.+ * dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size.+ * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)+ * If destination buffer is not large enough, decoding will stop and output an error code (negative value).+ * If the source stream is detected malformed, the function will stop decoding and return a negative result.+ * Note 1 : This function is protected against malicious data packets :+ * it will never writes outside 'dst' buffer, nor read outside 'source' buffer,+ * even if the compressed block is maliciously modified to order the decoder to do these actions.+ * In such case, the decoder stops immediately, and considers the compressed block malformed.+ * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them.+ * The implementation is free to send / store / derive this information in whichever way is most beneficial.+ * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead.+ */+LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);+++/*-************************************+* Advanced Functions+**************************************/+#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */+#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)++/*! LZ4_compressBound() :+ Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)+ This function is primarily useful for memory allocation purposes (destination buffer size).+ Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).+ Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize)+ inputSize : max supported value is LZ4_MAX_INPUT_SIZE+ return : maximum output size in a "worst case" scenario+ or 0, if input size is incorrect (too large or negative)+*/+LZ4LIB_API int LZ4_compressBound(int inputSize);++/*! LZ4_compress_fast() :+ Same as LZ4_compress_default(), but allows selection of "acceleration" factor.+ The larger the acceleration value, the faster the algorithm, but also the lesser the compression.+ It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.+ An acceleration value of "1" is the same as regular LZ4_compress_default()+ Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c).+*/+LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);+++/*! LZ4_compress_fast_extState() :+ * Same as LZ4_compress_fast(), using an externally allocated memory space for its state.+ * Use LZ4_sizeofState() to know how much memory must be allocated,+ * and allocate it on 8-bytes boundaries (using `malloc()` typically).+ * Then, provide this buffer as `void* state` to compression function.+ */+LZ4LIB_API int LZ4_sizeofState(void);+LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);+++/*! LZ4_compress_destSize() :+ * Reverse the logic : compresses as much data as possible from 'src' buffer+ * into already allocated buffer 'dst', of size >= 'targetDestSize'.+ * This function either compresses the entire 'src' content into 'dst' if it's large enough,+ * or fill 'dst' buffer completely with as much data as possible from 'src'.+ * note: acceleration parameter is fixed to "default".+ *+ * *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.+ * New value is necessarily <= input value.+ * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize)+ * or 0 if compression fails.+*/+LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);+++/*! LZ4_decompress_safe_partial() :+ * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src',+ * into destination buffer 'dst' of size 'dstCapacity'.+ * Up to 'targetOutputSize' bytes will be decoded.+ * The function stops decoding on reaching this objective,+ * which can boost performance when only the beginning of a block is required.+ *+ * @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity)+ * If source stream is detected malformed, function returns a negative result.+ *+ * Note : @return can be < targetOutputSize, if compressed block contains less data.+ *+ * Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity,+ * and expects targetOutputSize <= dstCapacity.+ * It effectively stops decoding on reaching targetOutputSize,+ * so dstCapacity is kind of redundant.+ * This is because in a previous version of this function,+ * decoding operation would not "break" a sequence in the middle.+ * As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize,+ * it could write more bytes, though only up to dstCapacity.+ * Some "margin" used to be required for this operation to work properly.+ * This is no longer necessary.+ * The function nonetheless keeps its signature, in an effort to not break API.+ */+LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);+++/*-*********************************************+* Streaming Compression Functions+***********************************************/+typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */++LZ4LIB_API LZ4_stream_t* LZ4_createStream(void);+LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr);++/*! LZ4_resetStream_fast() : v1.9.0++ * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks+ * (e.g., LZ4_compress_fast_continue()).+ *+ * An LZ4_stream_t must be initialized once before usage.+ * This is automatically done when created by LZ4_createStream().+ * However, should the LZ4_stream_t be simply declared on stack (for example),+ * it's necessary to initialize it first, using LZ4_initStream().+ *+ * After init, start any new stream with LZ4_resetStream_fast().+ * A same LZ4_stream_t can be re-used multiple times consecutively+ * and compress multiple streams,+ * provided that it starts each new stream with LZ4_resetStream_fast().+ *+ * LZ4_resetStream_fast() is much faster than LZ4_initStream(),+ * but is not compatible with memory regions containing garbage data.+ *+ * Note: it's only useful to call LZ4_resetStream_fast()+ * in the context of streaming compression.+ * The *extState* functions perform their own resets.+ * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive.+ */+LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);++/*! LZ4_loadDict() :+ * Use this function to reference a static dictionary into LZ4_stream_t.+ * The dictionary must remain available during compression.+ * LZ4_loadDict() triggers a reset, so any previous data will be forgotten.+ * The same dictionary will have to be loaded on decompression side for successful decoding.+ * Dictionary are useful for better compression of small data (KB range).+ * While LZ4 accept any input as dictionary,+ * results are generally better when using Zstandard's Dictionary Builder.+ * Loading a size of 0 is allowed, and is the same as reset.+ * @return : loaded dictionary size, in bytes (necessarily <= 64 KB)+ */+LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);++/*! LZ4_compress_fast_continue() :+ * Compress 'src' content using data from previously compressed blocks, for better compression ratio.+ * 'dst' buffer must be already allocated.+ * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.+ *+ * @return : size of compressed block+ * or 0 if there is an error (typically, cannot fit into 'dst').+ *+ * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block.+ * Each block has precise boundaries.+ * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata.+ * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together.+ *+ * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory !+ *+ * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB.+ * Make sure that buffers are separated, by at least one byte.+ * This construction ensures that each block only depends on previous block.+ *+ * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.+ *+ * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed.+ */+LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);++/*! LZ4_saveDict() :+ * If last 64KB data cannot be guaranteed to remain available at its current memory location,+ * save it into a safer place (char* safeBuffer).+ * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(),+ * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables.+ * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error.+ */+LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize);+++/*-**********************************************+* Streaming Decompression Functions+* Bufferless synchronous API+************************************************/+typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */++/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() :+ * creation / destruction of streaming decompression tracking context.+ * A tracking context can be re-used multiple times.+ */+LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);+LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);++/*! LZ4_setStreamDecode() :+ * An LZ4_streamDecode_t context can be allocated once and re-used multiple times.+ * Use this function to start decompression of a new stream of blocks.+ * A dictionary can optionally be set. Use NULL or size 0 for a reset order.+ * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.+ * @return : 1 if OK, 0 if error+ */+LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);++/*! LZ4_decoderRingBufferSize() : v1.8.2++ * Note : in a ring buffer scenario (optional),+ * blocks are presumed decompressed next to each other+ * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),+ * at which stage it resumes from beginning of ring buffer.+ * When setting such a ring buffer for streaming decompression,+ * provides the minimum size of this ring buffer+ * to be compatible with any source respecting maxBlockSize condition.+ * @return : minimum ring buffer size,+ * or 0 if there is an error (invalid maxBlockSize).+ */+LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize);+#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */++/*! LZ4_decompress_*_continue() :+ * These decoding functions allow decompression of consecutive blocks in "streaming" mode.+ * A block is an unsplittable entity, it must be presented entirely to a decompression function.+ * Decompression functions only accepts one block at a time.+ * The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded.+ * If less than 64KB of data has been decoded, all the data must be present.+ *+ * Special : if decompression side sets a ring buffer, it must respect one of the following conditions :+ * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize).+ * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes.+ * In which case, encoding and decoding buffers do not need to be synchronized.+ * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize.+ * - Synchronized mode :+ * Decompression buffer size is _exactly_ the same as compression buffer size,+ * and follows exactly same update rule (block boundaries at same positions),+ * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream),+ * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB).+ * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes.+ * In which case, encoding and decoding buffers do not need to be synchronized,+ * and encoding ring buffer can have any size, including small ones ( < 64 KB).+ *+ * Whenever these conditions are not possible,+ * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,+ * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.+*/+LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);+++/*! LZ4_decompress_*_usingDict() :+ * These decoding functions work the same as+ * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()+ * They are stand-alone, and don't need an LZ4_streamDecode_t structure.+ * Dictionary is presumed stable : it must remain accessible and unmodified during decompression.+ * Performance tip : Decompression speed can be substantially increased+ * when dst == dictStart + dictSize.+ */+LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);++#endif /* LZ4_H_2983827168210 */+++/*^*************************************+ * !!!!!! STATIC LINKING ONLY !!!!!!+ ***************************************/++/*-****************************************************************************+ * Experimental section+ *+ * Symbols declared in this section must be considered unstable. Their+ * signatures or semantics may change, or they may be removed altogether in the+ * future. They are therefore only safe to depend on when the caller is+ * statically linked against the library.+ *+ * To protect against unsafe usage, not only are the declarations guarded,+ * the definitions are hidden by default+ * when building LZ4 as a shared/dynamic library.+ *+ * In order to access these declarations,+ * define LZ4_STATIC_LINKING_ONLY in your application+ * before including LZ4's headers.+ *+ * In order to make their implementations accessible dynamically, you must+ * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library.+ ******************************************************************************/++#ifdef LZ4_STATIC_LINKING_ONLY++#ifndef LZ4_STATIC_3504398509+#define LZ4_STATIC_3504398509++#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS+#define LZ4LIB_STATIC_API LZ4LIB_API+#else+#define LZ4LIB_STATIC_API+#endif+++/*! LZ4_compress_fast_extState_fastReset() :+ * A variant of LZ4_compress_fast_extState().+ *+ * Using this variant avoids an expensive initialization step.+ * It is only safe to call if the state buffer is known to be correctly initialized already+ * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized").+ * From a high level, the difference is that+ * this function initializes the provided state with a call to something like LZ4_resetStream_fast()+ * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().+ */+LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);++/*! LZ4_attach_dictionary() :+ * This is an experimental API that allows+ * efficient use of a static dictionary many times.+ *+ * Rather than re-loading the dictionary buffer into a working context before+ * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a+ * working LZ4_stream_t, this function introduces a no-copy setup mechanism,+ * in which the working stream references the dictionary stream in-place.+ *+ * Several assumptions are made about the state of the dictionary stream.+ * Currently, only streams which have been prepared by LZ4_loadDict() should+ * be expected to work.+ *+ * Alternatively, the provided dictionaryStream may be NULL,+ * in which case any existing dictionary stream is unset.+ *+ * If a dictionary is provided, it replaces any pre-existing stream history.+ * The dictionary contents are the only history that can be referenced and+ * logically immediately precede the data compressed in the first subsequent+ * compression call.+ *+ * The dictionary will only remain attached to the working stream through the+ * first compression call, at the end of which it is cleared. The dictionary+ * stream (and source buffer) must remain in-place / accessible / unchanged+ * through the completion of the first compression call on the stream.+ */+LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);+++/*! In-place compression and decompression+ *+ * It's possible to have input and output sharing the same buffer,+ * for highly contrained memory environments.+ * In both cases, it requires input to lay at the end of the buffer,+ * and decompression to start at beginning of the buffer.+ * Buffer size must feature some margin, hence be larger than final size.+ *+ * |<------------------------buffer--------------------------------->|+ * |<-----------compressed data--------->|+ * |<-----------decompressed size------------------>|+ * |<----margin---->|+ *+ * This technique is more useful for decompression,+ * since decompressed size is typically larger,+ * and margin is short.+ *+ * In-place decompression will work inside any buffer+ * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize).+ * This presumes that decompressedSize > compressedSize.+ * Otherwise, it means compression actually expanded data,+ * and it would be more efficient to store such data with a flag indicating it's not compressed.+ * This can happen when data is not compressible (already compressed, or encrypted).+ *+ * For in-place compression, margin is larger, as it must be able to cope with both+ * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,+ * and data expansion, which can happen when input is not compressible.+ * As a consequence, buffer size requirements are much higher,+ * and memory savings offered by in-place compression are more limited.+ *+ * There are ways to limit this cost for compression :+ * - Reduce history size, by modifying LZ4_DISTANCE_MAX.+ * Note that it is a compile-time constant, so all compressions will apply this limit.+ * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,+ * so it's a reasonable trick when inputs are known to be small.+ * - Require the compressor to deliver a "maximum compressed size".+ * This is the `dstCapacity` parameter in `LZ4_compress*()`.+ * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,+ * in which case, the return code will be 0 (zero).+ * The caller must be ready for these cases to happen,+ * and typically design a backup scheme to send data uncompressed.+ * The combination of both techniques can significantly reduce+ * the amount of margin required for in-place compression.+ *+ * In-place compression can work in any buffer+ * which size is >= (maxCompressedSize)+ * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success.+ * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX,+ * so it's possible to reduce memory requirements by playing with them.+ */++#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32)+#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */++#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */+# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */+#endif++#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */+#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */++#endif /* LZ4_STATIC_3504398509 */+#endif /* LZ4_STATIC_LINKING_ONLY */++++#ifndef LZ4_H_98237428734687+#define LZ4_H_98237428734687++/*-************************************************************+ * PRIVATE DEFINITIONS+ **************************************************************+ * Do not use these definitions directly.+ * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.+ * Accessing members will expose code to API and/or ABI break in future versions of the library.+ **************************************************************/+#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)+#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)+#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */++#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)+#include <stdint.h>++typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;+struct LZ4_stream_t_internal {+ uint32_t hashTable[LZ4_HASH_SIZE_U32];+ uint32_t currentOffset;+ uint16_t dirty;+ uint16_t tableType;+ const uint8_t* dictionary;+ const LZ4_stream_t_internal* dictCtx;+ uint32_t dictSize;+};++typedef struct {+ const uint8_t* externalDict;+ size_t extDictSize;+ const uint8_t* prefixEnd;+ size_t prefixSize;+} LZ4_streamDecode_t_internal;++#else++typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;+struct LZ4_stream_t_internal {+ unsigned int hashTable[LZ4_HASH_SIZE_U32];+ unsigned int currentOffset;+ unsigned short dirty;+ unsigned short tableType;+ const unsigned char* dictionary;+ const LZ4_stream_t_internal* dictCtx;+ unsigned int dictSize;+};++typedef struct {+ const unsigned char* externalDict;+ const unsigned char* prefixEnd;+ size_t extDictSize;+ size_t prefixSize;+} LZ4_streamDecode_t_internal;++#endif++/*! LZ4_stream_t :+ * information structure to track an LZ4 stream.+ * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended.+ * The structure definition can be convenient for static allocation+ * (on stack, or as part of larger structure).+ * Init this structure with LZ4_initStream() before first use.+ * note : only use this definition in association with static linking !+ * this definition is not API/ABI safe, and may change in a future version.+ */+#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) /*AS-400*/ )+#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))+union LZ4_stream_u {+ unsigned long long table[LZ4_STREAMSIZE_U64];+ LZ4_stream_t_internal internal_donotuse;+} ; /* previously typedef'd to LZ4_stream_t */++/*! LZ4_initStream() : v1.9.0++ * An LZ4_stream_t structure must be initialized at least once.+ * This is automatically done when invoking LZ4_createStream(),+ * but it's not when the structure is simply declared on stack (for example).+ *+ * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t.+ * It can also initialize any arbitrary buffer of sufficient size,+ * and will @return a pointer of proper type upon initialization.+ *+ * Note : initialization fails if size and alignment conditions are not respected.+ * In which case, the function will @return NULL.+ * Note2: An LZ4_stream_t structure guarantees correct alignment and size.+ * Note3: Before v1.9.0, use LZ4_resetStream() instead+ */+LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size);+++/*! LZ4_streamDecode_t :+ * information structure to track an LZ4 stream during decompression.+ * init this structure using LZ4_setStreamDecode() before first use.+ * note : only use in association with static linking !+ * this definition is not API/ABI safe,+ * and may change in a future version !+ */+#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ )+#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))+union LZ4_streamDecode_u {+ unsigned long long table[LZ4_STREAMDECODESIZE_U64];+ LZ4_streamDecode_t_internal internal_donotuse;+} ; /* previously typedef'd to LZ4_streamDecode_t */++++/*-************************************+* Obsolete Functions+**************************************/++/*! Deprecation warnings+ *+ * Deprecated functions make the compiler generate a warning when invoked.+ * This is meant to invite users to update their source code.+ * Should deprecation warnings be a problem, it is generally possible to disable them,+ * typically with -Wno-deprecated-declarations for gcc+ * or _CRT_SECURE_NO_WARNINGS in Visual.+ *+ * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS+ * before including the header file.+ */+#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS+# define LZ4_DEPRECATED(message) /* disable deprecation warnings */+#else+# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)+# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */+# define LZ4_DEPRECATED(message) [[deprecated(message)]]+# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)+# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))+# elif (LZ4_GCC_VERSION >= 301)+# define LZ4_DEPRECATED(message) __attribute__((deprecated))+# elif defined(_MSC_VER)+# define LZ4_DEPRECATED(message) __declspec(deprecated(message))+# else+# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")+# define LZ4_DEPRECATED(message)+# endif+#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */++/* Obsolete compression functions */+LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize);+LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize);+LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);+LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);+LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);+LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);++/* Obsolete decompression functions */+LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);+LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);++/* Obsolete streaming functions; degraded functionality; do not use!+ *+ * In order to perform streaming compression, these functions depended on data+ * that is no longer tracked in the state. They have been preserved as well as+ * possible: using them will still produce a correct output. However, they don't+ * actually retain any history between compression calls. The compression ratio+ * achieved will therefore be no better than compressing each chunk+ * independently.+ */+LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);+LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void);+LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer);+LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state);++/* Obsolete streaming decoding functions */+LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);+LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);++/*! LZ4_decompress_fast() : **unsafe!**+ * These functions used to be faster than LZ4_decompress_safe(),+ * but it has changed, and they are now slower than LZ4_decompress_safe().+ * This is because LZ4_decompress_fast() doesn't know the input size,+ * and therefore must progress more cautiously in the input buffer to not read beyond the end of block.+ * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.+ * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.+ *+ * The last remaining LZ4_decompress_fast() specificity is that+ * it can decompress a block without knowing its compressed size.+ * Such functionality could be achieved in a more secure manner,+ * by also providing the maximum size of input buffer,+ * but it would require new prototypes, and adaptation of the implementation to this new use case.+ *+ * Parameters:+ * originalSize : is the uncompressed size to regenerate.+ * `dst` must be already allocated, its size must be >= 'originalSize' bytes.+ * @return : number of bytes read from source buffer (== compressed size).+ * The function expects to finish at block's end exactly.+ * If the source stream is detected malformed, the function stops decoding and returns a negative result.+ * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer.+ * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds.+ * Also, since match offsets are not validated, match reads from 'src' may underflow too.+ * These issues never happen if input (compressed) data is correct.+ * But they may happen if input data is invalid (error or intentional tampering).+ * As a consequence, use these functions in trusted environments with trusted data **only**.+ */++LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead")+LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);+LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead")+LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);+LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead")+LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);++/*! LZ4_resetStream() :+ * An LZ4_stream_t structure must be initialized at least once.+ * This is done with LZ4_initStream(), or LZ4_resetStream().+ * Consider switching to LZ4_initStream(),+ * invoking LZ4_resetStream() will trigger deprecation warnings in the future.+ */+LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);+++#endif /* LZ4_H_98237428734687 */+++#if defined (__cplusplus)+}+#endif
+ lz4/lib/lz4frame.c view
@@ -0,0 +1,1860 @@+/*+ * LZ4 auto-framing library+ * Copyright (C) 2011-2016, Yann Collet.+ *+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions are+ * met:+ *+ * - Redistributions of source code must retain the above copyright+ * notice, this list of conditions and the following disclaimer.+ * - Redistributions in binary form must reproduce the above+ * copyright notice, this list of conditions and the following disclaimer+ * in the documentation and/or other materials provided with the+ * distribution.+ *+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ *+ * You can contact the author at :+ * - LZ4 homepage : http://www.lz4.org+ * - LZ4 source repository : https://github.com/lz4/lz4+ */++/* LZ4F is a stand-alone API to create LZ4-compressed Frames+ * in full conformance with specification v1.6.1 .+ * This library rely upon memory management capabilities (malloc, free)+ * provided either by <stdlib.h>,+ * or redirected towards another library of user's choice+ * (see Memory Routines below).+ */+++/*-************************************+* Compiler Options+**************************************/+#ifdef _MSC_VER /* Visual Studio */+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */+#endif+++/*-************************************+* Tuning parameters+**************************************/+/*+ * LZ4F_HEAPMODE :+ * Select how default compression functions will allocate memory for their hash table,+ * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).+ */+#ifndef LZ4F_HEAPMODE+# define LZ4F_HEAPMODE 0+#endif+++/*-************************************+* Memory routines+**************************************/+/*+ * User may redirect invocations of+ * malloc(), calloc() and free()+ * towards another library or solution of their choice+ * by modifying below section.+ */+#include <stdlib.h> /* malloc, calloc, free */+#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */+# define ALLOC(s) malloc(s)+# define ALLOC_AND_ZERO(s) calloc(1,(s))+# define FREEMEM(p) free(p)+#endif++#include <string.h> /* memset, memcpy, memmove */+#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */+# define MEM_INIT(p,v,s) memset((p),(v),(s))+#endif+++/*-************************************+* Library declarations+**************************************/+#define LZ4F_STATIC_LINKING_ONLY+#include "lz4frame.h"+#define LZ4_STATIC_LINKING_ONLY+#include "lz4.h"+#define LZ4_HC_STATIC_LINKING_ONLY+#include "lz4hc.h"+#define XXH_STATIC_LINKING_ONLY+#include "xxhash.h"+++/*-************************************+* Debug+**************************************/+#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)+# include <assert.h>+#else+# ifndef assert+# define assert(condition) ((void)0)+# endif+#endif++#define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */++#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) && !defined(DEBUGLOG)+# include <stdio.h>+static int g_debuglog_enable = 1;+# define DEBUGLOG(l, ...) { \+ if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \+ fprintf(stderr, __FILE__ ": "); \+ fprintf(stderr, __VA_ARGS__); \+ fprintf(stderr, " \n"); \+ } }+#else+# define DEBUGLOG(l, ...) {} /* disabled */+#endif+++/*-************************************+* Basic Types+**************************************/+#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )+# include <stdint.h>+ typedef uint8_t BYTE;+ typedef uint16_t U16;+ typedef uint32_t U32;+ typedef int32_t S32;+ typedef uint64_t U64;+#else+ typedef unsigned char BYTE;+ typedef unsigned short U16;+ typedef unsigned int U32;+ typedef signed int S32;+ typedef unsigned long long U64;+#endif+++/* unoptimized version; solves endianess & alignment issues */+static U32 LZ4F_readLE32 (const void* src)+{+ const BYTE* const srcPtr = (const BYTE*)src;+ U32 value32 = srcPtr[0];+ value32 += ((U32)srcPtr[1])<< 8;+ value32 += ((U32)srcPtr[2])<<16;+ value32 += ((U32)srcPtr[3])<<24;+ return value32;+}++static void LZ4F_writeLE32 (void* dst, U32 value32)+{+ BYTE* const dstPtr = (BYTE*)dst;+ dstPtr[0] = (BYTE)value32;+ dstPtr[1] = (BYTE)(value32 >> 8);+ dstPtr[2] = (BYTE)(value32 >> 16);+ dstPtr[3] = (BYTE)(value32 >> 24);+}++static U64 LZ4F_readLE64 (const void* src)+{+ const BYTE* const srcPtr = (const BYTE*)src;+ U64 value64 = srcPtr[0];+ value64 += ((U64)srcPtr[1]<<8);+ value64 += ((U64)srcPtr[2]<<16);+ value64 += ((U64)srcPtr[3]<<24);+ value64 += ((U64)srcPtr[4]<<32);+ value64 += ((U64)srcPtr[5]<<40);+ value64 += ((U64)srcPtr[6]<<48);+ value64 += ((U64)srcPtr[7]<<56);+ return value64;+}++static void LZ4F_writeLE64 (void* dst, U64 value64)+{+ BYTE* const dstPtr = (BYTE*)dst;+ dstPtr[0] = (BYTE)value64;+ dstPtr[1] = (BYTE)(value64 >> 8);+ dstPtr[2] = (BYTE)(value64 >> 16);+ dstPtr[3] = (BYTE)(value64 >> 24);+ dstPtr[4] = (BYTE)(value64 >> 32);+ dstPtr[5] = (BYTE)(value64 >> 40);+ dstPtr[6] = (BYTE)(value64 >> 48);+ dstPtr[7] = (BYTE)(value64 >> 56);+}+++/*-************************************+* Constants+**************************************/+#ifndef LZ4_SRC_INCLUDED /* avoid double definition */+# define KB *(1<<10)+# define MB *(1<<20)+# define GB *(1<<30)+#endif++#define _1BIT 0x01+#define _2BITS 0x03+#define _3BITS 0x07+#define _4BITS 0x0F+#define _8BITS 0xFF++#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U+#define LZ4F_MAGICNUMBER 0x184D2204U+#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U+#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB++static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */+static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */+static const size_t BHSize = LZ4F_BLOCK_HEADER_SIZE; /* block header : size, and compress flag */+static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checksum (optional) */+++/*-************************************+* Structures and local types+**************************************/+typedef struct LZ4F_cctx_s+{+ LZ4F_preferences_t prefs;+ U32 version;+ U32 cStage;+ const LZ4F_CDict* cdict;+ size_t maxBlockSize;+ size_t maxBufferSize;+ BYTE* tmpBuff;+ BYTE* tmpIn;+ size_t tmpInSize;+ U64 totalInSize;+ XXH32_state_t xxh;+ void* lz4CtxPtr;+ U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */+ U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */+} LZ4F_cctx_t;+++/*-************************************+* Error management+**************************************/+#define LZ4F_GENERATE_STRING(STRING) #STRING,+static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) };+++unsigned LZ4F_isError(LZ4F_errorCode_t code)+{+ return (code > (LZ4F_errorCode_t)(-LZ4F_ERROR_maxCode));+}++const char* LZ4F_getErrorName(LZ4F_errorCode_t code)+{+ static const char* codeError = "Unspecified error code";+ if (LZ4F_isError(code)) return LZ4F_errorStrings[-(int)(code)];+ return codeError;+}++LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)+{+ if (!LZ4F_isError(functionResult)) return LZ4F_OK_NoError;+ return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult);+}++static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)+{+ /* A compilation error here means sizeof(ptrdiff_t) is not large enough */+ LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));+ return (LZ4F_errorCode_t)-(ptrdiff_t)code;+}++unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }++int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }++size_t LZ4F_getBlockSize(unsigned blockSizeID)+{+ static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB };++ if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;+ if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB)+ return err0r(LZ4F_ERROR_maxBlockSize_invalid);+ blockSizeID -= LZ4F_max64KB;+ return blockSizes[blockSizeID];+}++/*-************************************+* Private functions+**************************************/+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )++static BYTE LZ4F_headerChecksum (const void* header, size_t length)+{+ U32 const xxh = XXH32(header, length, 0);+ return (BYTE)(xxh >> 8);+}+++/*-************************************+* Simple-pass compression functions+**************************************/+static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID,+ const size_t srcSize)+{+ LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB;+ size_t maxBlockSize = 64 KB;+ while (requestedBSID > proposedBSID) {+ if (srcSize <= maxBlockSize)+ return proposedBSID;+ proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1);+ maxBlockSize <<= 2;+ }+ return requestedBSID;+}++/*! LZ4F_compressBound_internal() :+ * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.+ * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.+ * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.+ * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.+ */+static size_t LZ4F_compressBound_internal(size_t srcSize,+ const LZ4F_preferences_t* preferencesPtr,+ size_t alreadyBuffered)+{+ LZ4F_preferences_t prefsNull = LZ4F_INIT_PREFERENCES;+ prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */+ prefsNull.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; /* worst case */+ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;+ U32 const flush = prefsPtr->autoFlush | (srcSize==0);+ LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID;+ size_t const blockSize = LZ4F_getBlockSize(blockID);+ size_t const maxBuffered = blockSize - 1;+ size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered);+ size_t const maxSrcSize = srcSize + bufferedSize;+ unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize);+ size_t const partialBlockSize = maxSrcSize & (blockSize-1);+ size_t const lastBlockSize = flush ? partialBlockSize : 0;+ unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0);++ size_t const blockCRCSize = BFSize * prefsPtr->frameInfo.blockChecksumFlag;+ size_t const frameEnd = BHSize + (prefsPtr->frameInfo.contentChecksumFlag*BFSize);++ return ((BHSize + blockCRCSize) * nbBlocks) ++ (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;+ }+}++size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)+{+ LZ4F_preferences_t prefs;+ size_t const headerSize = maxFHSize; /* max header size, including optional fields */++ if (preferencesPtr!=NULL) prefs = *preferencesPtr;+ else MEM_INIT(&prefs, 0, sizeof(prefs));+ prefs.autoFlush = 1;++ return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);;+}+++/*! LZ4F_compressFrame_usingCDict() :+ * Compress srcBuffer using a dictionary, in a single step.+ * cdict can be NULL, in which case, no dictionary is used.+ * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).+ * The LZ4F_preferences_t structure is optional : you may provide NULL as argument,+ * however, it's the only way to provide a dictID, so it's not recommended.+ * @return : number of bytes written into dstBuffer,+ * or an error code if it fails (can be tested using LZ4F_isError())+ */+size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,+ void* dstBuffer, size_t dstCapacity,+ const void* srcBuffer, size_t srcSize,+ const LZ4F_CDict* cdict,+ const LZ4F_preferences_t* preferencesPtr)+{+ LZ4F_preferences_t prefs;+ LZ4F_compressOptions_t options;+ BYTE* const dstStart = (BYTE*) dstBuffer;+ BYTE* dstPtr = dstStart;+ BYTE* const dstEnd = dstStart + dstCapacity;++ if (preferencesPtr!=NULL)+ prefs = *preferencesPtr;+ else+ MEM_INIT(&prefs, 0, sizeof(prefs));+ if (prefs.frameInfo.contentSize != 0)+ prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */++ prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);+ prefs.autoFlush = 1;+ if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))+ prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */++ MEM_INIT(&options, 0, sizeof(options));+ options.stableSrc = 1;++ if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */+ return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);++ { size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */+ if (LZ4F_isError(headerSize)) return headerSize;+ dstPtr += headerSize; /* header size */ }++ assert(dstEnd >= dstPtr);+ { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options);+ if (LZ4F_isError(cSize)) return cSize;+ dstPtr += cSize; }++ assert(dstEnd >= dstPtr);+ { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */+ if (LZ4F_isError(tailSize)) return tailSize;+ dstPtr += tailSize; }++ assert(dstEnd >= dstStart);+ return (size_t)(dstPtr - dstStart);+}+++/*! LZ4F_compressFrame() :+ * Compress an entire srcBuffer into a valid LZ4 frame, in a single step.+ * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).+ * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.+ * @return : number of bytes written into dstBuffer.+ * or an error code if it fails (can be tested using LZ4F_isError())+ */+size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,+ const void* srcBuffer, size_t srcSize,+ const LZ4F_preferences_t* preferencesPtr)+{+ size_t result;+#if (LZ4F_HEAPMODE)+ LZ4F_cctx_t *cctxPtr;+ result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);+ if (LZ4F_isError(result)) return result;+#else+ LZ4F_cctx_t cctx;+ LZ4_stream_t lz4ctx;+ LZ4F_cctx_t *cctxPtr = &cctx;++ DEBUGLOG(4, "LZ4F_compressFrame");+ MEM_INIT(&cctx, 0, sizeof(cctx));+ cctx.version = LZ4F_VERSION;+ cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */+ if (preferencesPtr == NULL ||+ preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN)+ {+ LZ4_initStream(&lz4ctx, sizeof(lz4ctx));+ cctxPtr->lz4CtxPtr = &lz4ctx;+ cctxPtr->lz4CtxAlloc = 1;+ cctxPtr->lz4CtxState = 1;+ }+#endif++ result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity,+ srcBuffer, srcSize,+ NULL, preferencesPtr);++#if (LZ4F_HEAPMODE)+ LZ4F_freeCompressionContext(cctxPtr);+#else+ if (preferencesPtr != NULL &&+ preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN)+ {+ FREEMEM(cctxPtr->lz4CtxPtr);+ }+#endif+ return result;+}+++/*-***************************************************+* Dictionary compression+*****************************************************/++struct LZ4F_CDict_s {+ void* dictContent;+ LZ4_stream_t* fastCtx;+ LZ4_streamHC_t* HCCtx;+}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */++/*! LZ4F_createCDict() :+ * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.+ * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.+ * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.+ * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict+ * @return : digested dictionary for compression, or NULL if failed */+LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)+{+ const char* dictStart = (const char*)dictBuffer;+ LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));+ DEBUGLOG(4, "LZ4F_createCDict");+ if (!cdict) return NULL;+ if (dictSize > 64 KB) {+ dictStart += dictSize - 64 KB;+ dictSize = 64 KB;+ }+ cdict->dictContent = ALLOC(dictSize);+ cdict->fastCtx = LZ4_createStream();+ cdict->HCCtx = LZ4_createStreamHC();+ if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {+ LZ4F_freeCDict(cdict);+ return NULL;+ }+ memcpy(cdict->dictContent, dictStart, dictSize);+ LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);+ LZ4_setCompressionLevel(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);+ LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);+ return cdict;+}++void LZ4F_freeCDict(LZ4F_CDict* cdict)+{+ if (cdict==NULL) return; /* support free on NULL */+ FREEMEM(cdict->dictContent);+ LZ4_freeStream(cdict->fastCtx);+ LZ4_freeStreamHC(cdict->HCCtx);+ FREEMEM(cdict);+}+++/*-*********************************+* Advanced compression functions+***********************************/++/*! LZ4F_createCompressionContext() :+ * The first thing to do is to create a compressionContext object, which will be used in all compression operations.+ * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.+ * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries.+ * The function will provide a pointer to an allocated LZ4F_compressionContext_t object.+ * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.+ * Object can release its memory using LZ4F_freeCompressionContext();+ */+LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)+{+ LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t));+ if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);++ cctxPtr->version = version;+ cctxPtr->cStage = 0; /* Next stage : init stream */++ *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr;++ return LZ4F_OK_NoError;+}+++LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext)+{+ LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext;++ if (cctxPtr != NULL) { /* support free on NULL */+ FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */+ FREEMEM(cctxPtr->tmpBuff);+ FREEMEM(LZ4F_compressionContext);+ }++ return LZ4F_OK_NoError;+}+++/**+ * This function prepares the internal LZ4(HC) stream for a new compression,+ * resetting the context and attaching the dictionary, if there is one.+ *+ * It needs to be called at the beginning of each independent compression+ * stream (i.e., at the beginning of a frame in blockLinked mode, or at the+ * beginning of each block in blockIndependent mode).+ */+static void LZ4F_initStream(void* ctx,+ const LZ4F_CDict* cdict,+ int level,+ LZ4F_blockMode_t blockMode) {+ if (level < LZ4HC_CLEVEL_MIN) {+ if (cdict != NULL || blockMode == LZ4F_blockLinked) {+ /* In these cases, we will call LZ4_compress_fast_continue(),+ * which needs an already reset context. Otherwise, we'll call a+ * one-shot API. The non-continued APIs internally perform their own+ * resets at the beginning of their calls, where they know what+ * tableType they need the context to be in. So in that case this+ * would be misguided / wasted work. */+ LZ4_resetStream_fast((LZ4_stream_t*)ctx);+ }+ LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);+ } else {+ LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);+ LZ4_attach_HC_dictionary((LZ4_streamHC_t *)ctx, cdict ? cdict->HCCtx : NULL);+ }+}+++/*! LZ4F_compressBegin_usingCDict() :+ * init streaming compression and writes frame header into dstBuffer.+ * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.+ * @return : number of bytes written into dstBuffer for the header+ * or an error code (can be tested using LZ4F_isError())+ */+size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,+ void* dstBuffer, size_t dstCapacity,+ const LZ4F_CDict* cdict,+ const LZ4F_preferences_t* preferencesPtr)+{+ LZ4F_preferences_t prefNull;+ BYTE* const dstStart = (BYTE*)dstBuffer;+ BYTE* dstPtr = dstStart;+ BYTE* headerStart;++ if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);+ MEM_INIT(&prefNull, 0, sizeof(prefNull));+ if (preferencesPtr == NULL) preferencesPtr = &prefNull;+ cctxPtr->prefs = *preferencesPtr;++ /* Ctx Management */+ { U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;+ if (cctxPtr->lz4CtxAlloc < ctxTypeID) {+ FREEMEM(cctxPtr->lz4CtxPtr);+ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {+ cctxPtr->lz4CtxPtr = LZ4_createStream();+ } else {+ cctxPtr->lz4CtxPtr = LZ4_createStreamHC();+ }+ if (cctxPtr->lz4CtxPtr == NULL)+ return err0r(LZ4F_ERROR_allocation_failed);+ cctxPtr->lz4CtxAlloc = ctxTypeID;+ cctxPtr->lz4CtxState = ctxTypeID;+ } else if (cctxPtr->lz4CtxState != ctxTypeID) {+ /* otherwise, a sufficient buffer is allocated, but we need to+ * reset it to the correct context type */+ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {+ LZ4_initStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr, sizeof (LZ4_stream_t));+ } else {+ LZ4_initStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));+ LZ4_setCompressionLevel((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);+ }+ cctxPtr->lz4CtxState = ctxTypeID;+ }+ }++ /* Buffer Management */+ if (cctxPtr->prefs.frameInfo.blockSizeID == 0)+ cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;+ cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID);++ { size_t const requiredBuffSize = preferencesPtr->autoFlush ?+ ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 64 KB : 0) : /* only needs past data up to window size */+ cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 128 KB : 0);++ if (cctxPtr->maxBufferSize < requiredBuffSize) {+ cctxPtr->maxBufferSize = 0;+ FREEMEM(cctxPtr->tmpBuff);+ cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize);+ if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);+ cctxPtr->maxBufferSize = requiredBuffSize;+ } }+ cctxPtr->tmpIn = cctxPtr->tmpBuff;+ cctxPtr->tmpInSize = 0;+ (void)XXH32_reset(&(cctxPtr->xxh), 0);++ /* context init */+ cctxPtr->cdict = cdict;+ if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {+ /* frame init only for blockLinked : blockIndependent will be init at each block */+ LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel, LZ4F_blockLinked);+ }+ if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {+ LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);+ }++ /* Magic Number */+ LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);+ dstPtr += 4;+ headerStart = dstPtr;++ /* FLG Byte */+ *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */+ + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)+ + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)+ + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3)+ + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)+ + (cctxPtr->prefs.frameInfo.dictID > 0) );+ /* BD Byte */+ *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);+ /* Optional Frame content size field */+ if (cctxPtr->prefs.frameInfo.contentSize) {+ LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize);+ dstPtr += 8;+ cctxPtr->totalInSize = 0;+ }+ /* Optional dictionary ID field */+ if (cctxPtr->prefs.frameInfo.dictID) {+ LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);+ dstPtr += 4;+ }+ /* Header CRC Byte */+ *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));+ dstPtr++;++ cctxPtr->cStage = 1; /* header written, now request input data block */+ return (size_t)(dstPtr - dstStart);+}+++/*! LZ4F_compressBegin() :+ * init streaming compression and writes frame header into dstBuffer.+ * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.+ * preferencesPtr can be NULL, in which case default parameters are selected.+ * @return : number of bytes written into dstBuffer for the header+ * or an error code (can be tested using LZ4F_isError())+ */+size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr,+ void* dstBuffer, size_t dstCapacity,+ const LZ4F_preferences_t* preferencesPtr)+{+ return LZ4F_compressBegin_usingCDict(cctxPtr, dstBuffer, dstCapacity,+ NULL, preferencesPtr);+}+++/* LZ4F_compressBound() :+ * @return minimum capacity of dstBuffer for a given srcSize to handle worst case scenario.+ * LZ4F_preferences_t structure is optional : if NULL, preferences will be set to cover worst case scenario.+ * This function cannot fail.+ */+size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)+{+ return LZ4F_compressBound_internal(srcSize, preferencesPtr, (size_t)-1);+}+++typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict);+++/*! LZ4F_makeBlock():+ * compress a single block, add header and optional checksum.+ * assumption : dst buffer capacity is >= BHSize + srcSize + crcSize+ */+static size_t LZ4F_makeBlock(void* dst,+ const void* src, size_t srcSize,+ compressFunc_t compress, void* lz4ctx, int level,+ const LZ4F_CDict* cdict,+ LZ4F_blockChecksum_t crcFlag)+{+ BYTE* const cSizePtr = (BYTE*)dst;+ U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),+ (int)(srcSize), (int)(srcSize-1),+ level, cdict);+ if (cSize == 0) { /* compression failed */+ cSize = (U32)srcSize;+ LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);+ memcpy(cSizePtr+BHSize, src, srcSize);+ } else {+ LZ4F_writeLE32(cSizePtr, cSize);+ }+ if (crcFlag) {+ U32 const crc32 = XXH32(cSizePtr+BHSize, cSize, 0); /* checksum of compressed data */+ LZ4F_writeLE32(cSizePtr+BHSize+cSize, crc32);+ }+ return BHSize + cSize + ((U32)crcFlag)*BFSize;+}+++static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)+{+ int const acceleration = (level < 0) ? -level + 1 : 1;+ LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);+ if (cdict) {+ return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);+ } else {+ return LZ4_compress_fast_extState_fastReset(ctx, src, dst, srcSize, dstCapacity, acceleration);+ }+}++static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)+{+ int const acceleration = (level < 0) ? -level + 1 : 1;+ (void)cdict; /* init once at beginning of frame */+ return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);+}++static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)+{+ LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);+ if (cdict) {+ return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);+ }+ return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level);+}++static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)+{+ (void)level; (void)cdict; /* init once at beginning of frame */+ return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);+}++static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)+{+ if (level < LZ4HC_CLEVEL_MIN) {+ if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;+ return LZ4F_compressBlock_continue;+ }+ if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC;+ return LZ4F_compressBlockHC_continue;+}++static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)+{+ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)+ return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);+ return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);+}++typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;++/*! LZ4F_compressUpdate() :+ * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.+ * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).+ * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.+ * or an error code if it fails (which can be tested using LZ4F_isError())+ */+size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,+ void* dstBuffer, size_t dstCapacity,+ const void* srcBuffer, size_t srcSize,+ const LZ4F_compressOptions_t* compressOptionsPtr)+{+ LZ4F_compressOptions_t cOptionsNull;+ size_t const blockSize = cctxPtr->maxBlockSize;+ const BYTE* srcPtr = (const BYTE*)srcBuffer;+ const BYTE* const srcEnd = srcPtr + srcSize;+ BYTE* const dstStart = (BYTE*)dstBuffer;+ BYTE* dstPtr = dstStart;+ LZ4F_lastBlockStatus lastBlockCompressed = notDone;+ compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);++ DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);++ if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);+ if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))+ return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);+ MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull));+ if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;++ /* complete tmp buffer */+ if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */+ size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;+ if (sizeToCopy > srcSize) {+ /* add src to tmpIn buffer */+ memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);+ srcPtr = srcEnd;+ cctxPtr->tmpInSize += srcSize;+ /* still needs some CRC */+ } else {+ /* complete tmpIn block and then compress it */+ lastBlockCompressed = fromTmpBuffer;+ memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);+ srcPtr += sizeToCopy;++ dstPtr += LZ4F_makeBlock(dstPtr,+ cctxPtr->tmpIn, blockSize,+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,+ cctxPtr->cdict,+ cctxPtr->prefs.frameInfo.blockChecksumFlag);++ if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;+ cctxPtr->tmpInSize = 0;+ }+ }++ while ((size_t)(srcEnd - srcPtr) >= blockSize) {+ /* compress full blocks */+ lastBlockCompressed = fromSrcBuffer;+ dstPtr += LZ4F_makeBlock(dstPtr,+ srcPtr, blockSize,+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,+ cctxPtr->cdict,+ cctxPtr->prefs.frameInfo.blockChecksumFlag);+ srcPtr += blockSize;+ }++ if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {+ /* compress remaining input < blockSize */+ lastBlockCompressed = fromSrcBuffer;+ dstPtr += LZ4F_makeBlock(dstPtr,+ srcPtr, (size_t)(srcEnd - srcPtr),+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,+ cctxPtr->cdict,+ cctxPtr->prefs.frameInfo.blockChecksumFlag);+ srcPtr = srcEnd;+ }++ /* preserve dictionary if necessary */+ if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {+ if (compressOptionsPtr->stableSrc) {+ cctxPtr->tmpIn = cctxPtr->tmpBuff;+ } else {+ int const realDictSize = LZ4F_localSaveDict(cctxPtr);+ if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC);+ cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;+ }+ }++ /* keep tmpIn within limits */+ if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */+ && !(cctxPtr->prefs.autoFlush))+ {+ int const realDictSize = LZ4F_localSaveDict(cctxPtr);+ cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;+ }++ /* some input data left, necessarily < blockSize */+ if (srcPtr < srcEnd) {+ /* fill tmp buffer */+ size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);+ memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);+ cctxPtr->tmpInSize = sizeToCopy;+ }++ if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)+ (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);++ cctxPtr->totalInSize += srcSize;+ return (size_t)(dstPtr - dstStart);+}+++/*! LZ4F_flush() :+ * When compressed data must be sent immediately, without waiting for a block to be filled,+ * invoke LZ4_flush(), which will immediately compress any remaining data stored within LZ4F_cctx.+ * The result of the function is the number of bytes written into dstBuffer.+ * It can be zero, this means there was no data left within LZ4F_cctx.+ * The function outputs an error code if it fails (can be tested using LZ4F_isError())+ * LZ4F_compressOptions_t* is optional. NULL is a valid argument.+ */+size_t LZ4F_flush(LZ4F_cctx* cctxPtr,+ void* dstBuffer, size_t dstCapacity,+ const LZ4F_compressOptions_t* compressOptionsPtr)+{+ BYTE* const dstStart = (BYTE*)dstBuffer;+ BYTE* dstPtr = dstStart;+ compressFunc_t compress;++ if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */+ if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);+ if (dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize))+ return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);+ (void)compressOptionsPtr; /* not yet useful */++ /* select compression function */+ compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);++ /* compress tmp buffer */+ dstPtr += LZ4F_makeBlock(dstPtr,+ cctxPtr->tmpIn, cctxPtr->tmpInSize,+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,+ cctxPtr->cdict,+ cctxPtr->prefs.frameInfo.blockChecksumFlag);+ assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity));++ if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)+ cctxPtr->tmpIn += cctxPtr->tmpInSize;+ cctxPtr->tmpInSize = 0;++ /* keep tmpIn within limits */+ if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */+ int const realDictSize = LZ4F_localSaveDict(cctxPtr);+ cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;+ }++ return (size_t)(dstPtr - dstStart);+}+++/*! LZ4F_compressEnd() :+ * When you want to properly finish the compressed frame, just call LZ4F_compressEnd().+ * It will flush whatever data remained within compressionContext (like LZ4_flush())+ * but also properly finalize the frame, with an endMark and an (optional) checksum.+ * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.+ * @return: the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))+ * or an error code if it fails (can be tested using LZ4F_isError())+ * The context can then be used again to compress a new frame, starting with LZ4F_compressBegin().+ */+size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,+ void* dstBuffer, size_t dstCapacity,+ const LZ4F_compressOptions_t* compressOptionsPtr)+{+ BYTE* const dstStart = (BYTE*)dstBuffer;+ BYTE* dstPtr = dstStart;++ size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);+ if (LZ4F_isError(flushSize)) return flushSize;+ dstPtr += flushSize;++ assert(flushSize <= dstCapacity);+ dstCapacity -= flushSize;++ if (dstCapacity < 4) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);+ LZ4F_writeLE32(dstPtr, 0);+ dstPtr += 4; /* endMark */++ if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) {+ U32 const xxh = XXH32_digest(&(cctxPtr->xxh));+ if (dstCapacity < 8) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);+ LZ4F_writeLE32(dstPtr, xxh);+ dstPtr+=4; /* content Checksum */+ }++ cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */+ cctxPtr->maxBufferSize = 0; /* reuse HC context */++ if (cctxPtr->prefs.frameInfo.contentSize) {+ if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize)+ return err0r(LZ4F_ERROR_frameSize_wrong);+ }++ return (size_t)(dstPtr - dstStart);+}+++/*-***************************************************+* Frame Decompression+*****************************************************/++typedef enum {+ dstage_getFrameHeader=0, dstage_storeFrameHeader,+ dstage_init,+ dstage_getBlockHeader, dstage_storeBlockHeader,+ dstage_copyDirect, dstage_getBlockChecksum,+ dstage_getCBlock, dstage_storeCBlock,+ dstage_flushOut,+ dstage_getSuffix, dstage_storeSuffix,+ dstage_getSFrameSize, dstage_storeSFrameSize,+ dstage_skipSkippable+} dStage_t;++struct LZ4F_dctx_s {+ LZ4F_frameInfo_t frameInfo;+ U32 version;+ dStage_t dStage;+ U64 frameRemainingSize;+ size_t maxBlockSize;+ size_t maxBufferSize;+ BYTE* tmpIn;+ size_t tmpInSize;+ size_t tmpInTarget;+ BYTE* tmpOutBuffer;+ const BYTE* dict;+ size_t dictSize;+ BYTE* tmpOut;+ size_t tmpOutSize;+ size_t tmpOutStart;+ XXH32_state_t xxh;+ XXH32_state_t blockChecksum;+ BYTE header[LZ4F_HEADER_SIZE_MAX];+}; /* typedef'd to LZ4F_dctx in lz4frame.h */+++/*! LZ4F_createDecompressionContext() :+ * Create a decompressionContext object, which will track all decompression operations.+ * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.+ * Object can later be released using LZ4F_freeDecompressionContext().+ * @return : if != 0, there was an error during context creation.+ */+LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)+{+ LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx));+ if (dctx == NULL) { /* failed allocation */+ *LZ4F_decompressionContextPtr = NULL;+ return err0r(LZ4F_ERROR_allocation_failed);+ }++ dctx->version = versionNumber;+ *LZ4F_decompressionContextPtr = dctx;+ return LZ4F_OK_NoError;+}++LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx)+{+ LZ4F_errorCode_t result = LZ4F_OK_NoError;+ if (dctx != NULL) { /* can accept NULL input, like free() */+ result = (LZ4F_errorCode_t)dctx->dStage;+ FREEMEM(dctx->tmpIn);+ FREEMEM(dctx->tmpOutBuffer);+ FREEMEM(dctx);+ }+ return result;+}+++/*==--- Streaming Decompression operations ---==*/++void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)+{+ dctx->dStage = dstage_getFrameHeader;+ dctx->dict = NULL;+ dctx->dictSize = 0;+}+++/*! LZ4F_decodeHeader() :+ * input : `src` points at the **beginning of the frame**+ * output : set internal values of dctx, such as+ * dctx->frameInfo and dctx->dStage.+ * Also allocates internal buffers.+ * @return : nb Bytes read from src (necessarily <= srcSize)+ * or an error code (testable with LZ4F_isError())+ */+static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize)+{+ unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID;+ size_t frameHeaderSize;+ const BYTE* srcPtr = (const BYTE*)src;++ /* need to decode header to get frameInfo */+ if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */+ MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));++ /* special case : skippable frames */+ if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) {+ dctx->frameInfo.frameType = LZ4F_skippableFrame;+ if (src == (void*)(dctx->header)) {+ dctx->tmpInSize = srcSize;+ dctx->tmpInTarget = 8;+ dctx->dStage = dstage_storeSFrameSize;+ return srcSize;+ } else {+ dctx->dStage = dstage_getSFrameSize;+ return 4;+ }+ }++ /* control magic number */+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION+ if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER)+ return err0r(LZ4F_ERROR_frameType_unknown);+#endif+ dctx->frameInfo.frameType = LZ4F_frame;++ /* Flags */+ { U32 const FLG = srcPtr[4];+ U32 const version = (FLG>>6) & _2BITS;+ blockChecksumFlag = (FLG>>4) & _1BIT;+ blockMode = (FLG>>5) & _1BIT;+ contentSizeFlag = (FLG>>3) & _1BIT;+ contentChecksumFlag = (FLG>>2) & _1BIT;+ dictIDFlag = FLG & _1BIT;+ /* validate */+ if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */+ if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */+ }++ /* Frame Header Size */+ frameHeaderSize = minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0);++ if (srcSize < frameHeaderSize) {+ /* not enough input to fully decode frame header */+ if (srcPtr != dctx->header)+ memcpy(dctx->header, srcPtr, srcSize);+ dctx->tmpInSize = srcSize;+ dctx->tmpInTarget = frameHeaderSize;+ dctx->dStage = dstage_storeFrameHeader;+ return srcSize;+ }++ { U32 const BD = srcPtr[5];+ blockSizeID = (BD>>4) & _3BITS;+ /* validate */+ if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */+ if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */+ if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */+ }++ /* check header */+ assert(frameHeaderSize > 5);+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION+ { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);+ if (HC != srcPtr[frameHeaderSize-1])+ return err0r(LZ4F_ERROR_headerChecksum_invalid);+ }+#endif++ /* save */+ dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;+ dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag;+ dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;+ dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;+ dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID);+ if (contentSizeFlag)+ dctx->frameRemainingSize =+ dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);+ if (dictIDFlag)+ dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);++ dctx->dStage = dstage_init;++ return frameHeaderSize;+}+++/*! LZ4F_headerSize() :+ * @return : size of frame header+ * or an error code, which can be tested using LZ4F_isError()+ */+size_t LZ4F_headerSize(const void* src, size_t srcSize)+{+ if (src == NULL) return err0r(LZ4F_ERROR_srcPtr_wrong);++ /* minimal srcSize to determine header size */+ if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH)+ return err0r(LZ4F_ERROR_frameHeader_incomplete);++ /* special case : skippable frames */+ if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START)+ return 8;++ /* control magic number */+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION+ if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)+ return err0r(LZ4F_ERROR_frameType_unknown);+#endif++ /* Frame Header Size */+ { BYTE const FLG = ((const BYTE*)src)[4];+ U32 const contentSizeFlag = (FLG>>3) & _1BIT;+ U32 const dictIDFlag = FLG & _1BIT;+ return minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0);+ }+}++/*! LZ4F_getFrameInfo() :+ * This function extracts frame parameters (max blockSize, frame checksum, etc.).+ * Usage is optional. Objective is to provide relevant information for allocation purposes.+ * This function works in 2 situations :+ * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.+ * Amount of input data provided must be large enough to successfully decode the frame header.+ * A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum.+ * - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx.+ * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).+ * Decompression must resume from (srcBuffer + *srcSizePtr).+ * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,+ * or an error code which can be tested using LZ4F_isError()+ * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.+ * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.+ */+LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,+ LZ4F_frameInfo_t* frameInfoPtr,+ const void* srcBuffer, size_t* srcSizePtr)+{+ LZ4F_STATIC_ASSERT(dstage_getFrameHeader < dstage_storeFrameHeader);+ if (dctx->dStage > dstage_storeFrameHeader) {+ /* frameInfo already decoded */+ size_t o=0, i=0;+ *srcSizePtr = 0;+ *frameInfoPtr = dctx->frameInfo;+ /* returns : recommended nb of bytes for LZ4F_decompress() */+ return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL);+ } else {+ if (dctx->dStage == dstage_storeFrameHeader) {+ /* frame decoding already started, in the middle of header => automatic fail */+ *srcSizePtr = 0;+ return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);+ } else {+ size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);+ if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }+ if (*srcSizePtr < hSize) {+ *srcSizePtr=0;+ return err0r(LZ4F_ERROR_frameHeader_incomplete);+ }++ { size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);+ if (LZ4F_isError(decodeResult)) {+ *srcSizePtr = 0;+ } else {+ *srcSizePtr = decodeResult;+ decodeResult = BHSize; /* block header size */+ }+ *frameInfoPtr = dctx->frameInfo;+ return decodeResult;+ } } }+}+++/* LZ4F_updateDict() :+ * only used for LZ4F_blockLinked mode */+static void LZ4F_updateDict(LZ4F_dctx* dctx,+ const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart,+ unsigned withinTmp)+{+ if (dctx->dictSize==0)+ dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */++ if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity, directly within dstBuffer */+ dctx->dictSize += dstSize;+ return;+ }++ assert(dstPtr >= dstBufferStart);+ if ((size_t)(dstPtr - dstBufferStart) + dstSize >= 64 KB) { /* history in dstBuffer becomes large enough to become dictionary */+ dctx->dict = (const BYTE*)dstBufferStart;+ dctx->dictSize = (size_t)(dstPtr - dstBufferStart) + dstSize;+ return;+ }++ assert(dstSize < 64 KB); /* if dstSize >= 64 KB, dictionary would be set into dstBuffer directly */++ /* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOut */++ if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) { /* continue history within tmpOutBuffer */+ /* withinTmp expectation : content of [dstPtr,dstSize] is same as [dict+dictSize,dstSize], so we just extend it */+ assert(dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart);+ dctx->dictSize += dstSize;+ return;+ }++ if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */+ size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer);+ size_t copySize = 64 KB - dctx->tmpOutSize;+ const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;+ if (dctx->tmpOutSize > 64 KB) copySize = 0;+ if (copySize > preserveSize) copySize = preserveSize;++ memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);++ dctx->dict = dctx->tmpOutBuffer;+ dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize;+ return;+ }++ if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */+ if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */+ size_t const preserveSize = 64 KB - dstSize;+ memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);+ dctx->dictSize = preserveSize;+ }+ memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize);+ dctx->dictSize += dstSize;+ return;+ }++ /* join dict & dest into tmp */+ { size_t preserveSize = 64 KB - dstSize;+ if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize;+ memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);+ memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize);+ dctx->dict = dctx->tmpOutBuffer;+ dctx->dictSize = preserveSize + dstSize;+ }+}++++/*! LZ4F_decompress() :+ * Call this function repetitively to regenerate compressed data in srcBuffer.+ * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer+ * into dstBuffer of capacity *dstSizePtr.+ *+ * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).+ *+ * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).+ * If number of bytes read is < number of bytes provided, then decompression operation is not complete.+ * Remaining data will have to be presented again in a subsequent invocation.+ *+ * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.+ * Schematically, it's the size of the current (or remaining) compressed block + header of next block.+ * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling.+ * Note that this is just a hint, and it's always possible to any srcSize value.+ * When a frame is fully decoded, @return will be 0.+ * If decompression failed, @return is an error code which can be tested using LZ4F_isError().+ */+size_t LZ4F_decompress(LZ4F_dctx* dctx,+ void* dstBuffer, size_t* dstSizePtr,+ const void* srcBuffer, size_t* srcSizePtr,+ const LZ4F_decompressOptions_t* decompressOptionsPtr)+{+ LZ4F_decompressOptions_t optionsNull;+ const BYTE* const srcStart = (const BYTE*)srcBuffer;+ const BYTE* const srcEnd = srcStart + *srcSizePtr;+ const BYTE* srcPtr = srcStart;+ BYTE* const dstStart = (BYTE*)dstBuffer;+ BYTE* const dstEnd = dstStart + *dstSizePtr;+ BYTE* dstPtr = dstStart;+ const BYTE* selectedIn = NULL;+ unsigned doAnotherStage = 1;+ size_t nextSrcSizeHint = 1;+++ MEM_INIT(&optionsNull, 0, sizeof(optionsNull));+ if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull;+ *srcSizePtr = 0;+ *dstSizePtr = 0;++ /* behaves as a state machine */++ while (doAnotherStage) {++ switch(dctx->dStage)+ {++ case dstage_getFrameHeader:+ if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */+ size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */+ if (LZ4F_isError(hSize)) return hSize;+ srcPtr += hSize;+ break;+ }+ dctx->tmpInSize = 0;+ if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */+ dctx->tmpInTarget = minFHSize; /* minimum size to decode header */+ dctx->dStage = dstage_storeFrameHeader;+ /* fall-through */++ case dstage_storeFrameHeader:+ { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr));+ memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);+ dctx->tmpInSize += sizeToCopy;+ srcPtr += sizeToCopy;+ }+ if (dctx->tmpInSize < dctx->tmpInTarget) {+ nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */+ doAnotherStage = 0; /* not enough src data, ask for some more */+ break;+ }+ { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */+ if (LZ4F_isError(hSize)) return hSize;+ }+ break;++ case dstage_init:+ if (dctx->frameInfo.contentChecksumFlag) (void)XXH32_reset(&(dctx->xxh), 0);+ /* internal buffers allocation */+ { size_t const bufferNeeded = dctx->maxBlockSize+ + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0);+ if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */+ dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/+ FREEMEM(dctx->tmpIn);+ dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + BFSize /* block checksum */);+ if (dctx->tmpIn == NULL)+ return err0r(LZ4F_ERROR_allocation_failed);+ FREEMEM(dctx->tmpOutBuffer);+ dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded);+ if (dctx->tmpOutBuffer== NULL)+ return err0r(LZ4F_ERROR_allocation_failed);+ dctx->maxBufferSize = bufferNeeded;+ } }+ dctx->tmpInSize = 0;+ dctx->tmpInTarget = 0;+ dctx->tmpOut = dctx->tmpOutBuffer;+ dctx->tmpOutStart = 0;+ dctx->tmpOutSize = 0;++ dctx->dStage = dstage_getBlockHeader;+ /* fall-through */++ case dstage_getBlockHeader:+ if ((size_t)(srcEnd - srcPtr) >= BHSize) {+ selectedIn = srcPtr;+ srcPtr += BHSize;+ } else {+ /* not enough input to read cBlockSize field */+ dctx->tmpInSize = 0;+ dctx->dStage = dstage_storeBlockHeader;+ }++ if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */+ case dstage_storeBlockHeader:+ { size_t const remainingInput = (size_t)(srcEnd - srcPtr);+ size_t const wantedData = BHSize - dctx->tmpInSize;+ size_t const sizeToCopy = MIN(wantedData, remainingInput);+ memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);+ srcPtr += sizeToCopy;+ dctx->tmpInSize += sizeToCopy;++ if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */+ nextSrcSizeHint = BHSize - dctx->tmpInSize;+ doAnotherStage = 0;+ break;+ }+ selectedIn = dctx->tmpIn;+ } /* if (dctx->dStage == dstage_storeBlockHeader) */++ /* decode block header */+ { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;+ size_t const crcSize = dctx->frameInfo.blockChecksumFlag * BFSize;+ if (nextCBlockSize==0) { /* frameEnd signal, no more block */+ dctx->dStage = dstage_getSuffix;+ break;+ }+ if (nextCBlockSize > dctx->maxBlockSize)+ return err0r(LZ4F_ERROR_maxBlockSize_invalid);+ if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) {+ /* next block is uncompressed */+ dctx->tmpInTarget = nextCBlockSize;+ if (dctx->frameInfo.blockChecksumFlag) {+ (void)XXH32_reset(&dctx->blockChecksum, 0);+ }+ dctx->dStage = dstage_copyDirect;+ break;+ }+ /* next block is a compressed block */+ dctx->tmpInTarget = nextCBlockSize + crcSize;+ dctx->dStage = dstage_getCBlock;+ if (dstPtr==dstEnd || srcPtr==srcEnd) {+ nextSrcSizeHint = BHSize + nextCBlockSize + crcSize;+ doAnotherStage = 0;+ }+ break;+ }++ case dstage_copyDirect: /* uncompressed block */+ { size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));+ size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);+ memcpy(dstPtr, srcPtr, sizeToCopy);+ if (dctx->frameInfo.blockChecksumFlag) {+ (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);+ }+ if (dctx->frameInfo.contentChecksumFlag)+ (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);+ if (dctx->frameInfo.contentSize)+ dctx->frameRemainingSize -= sizeToCopy;++ /* history management (linked blocks only)*/+ if (dctx->frameInfo.blockMode == LZ4F_blockLinked)+ LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0);++ srcPtr += sizeToCopy;+ dstPtr += sizeToCopy;+ if (sizeToCopy == dctx->tmpInTarget) { /* all done */+ if (dctx->frameInfo.blockChecksumFlag) {+ dctx->tmpInSize = 0;+ dctx->dStage = dstage_getBlockChecksum;+ } else+ dctx->dStage = dstage_getBlockHeader; /* new block */+ break;+ }+ dctx->tmpInTarget -= sizeToCopy; /* need to copy more */+ nextSrcSizeHint = dctx->tmpInTarget ++ +(dctx->frameInfo.blockChecksumFlag ? BFSize : 0)+ + BHSize /* next header size */;+ doAnotherStage = 0;+ break;+ }++ /* check block checksum for recently transferred uncompressed block */+ case dstage_getBlockChecksum:+ { const void* crcSrc;+ if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) {+ crcSrc = srcPtr;+ srcPtr += 4;+ } else {+ size_t const stillToCopy = 4 - dctx->tmpInSize;+ size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr));+ memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);+ dctx->tmpInSize += sizeToCopy;+ srcPtr += sizeToCopy;+ if (dctx->tmpInSize < 4) { /* all input consumed */+ doAnotherStage = 0;+ break;+ }+ crcSrc = dctx->header;+ }+ { U32 const readCRC = LZ4F_readLE32(crcSrc);+ U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION+ if (readCRC != calcCRC)+ return err0r(LZ4F_ERROR_blockChecksum_invalid);+#else+ (void)readCRC;+ (void)calcCRC;+#endif+ } }+ dctx->dStage = dstage_getBlockHeader; /* new block */+ break;++ case dstage_getCBlock:+ if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) {+ dctx->tmpInSize = 0;+ dctx->dStage = dstage_storeCBlock;+ break;+ }+ /* input large enough to read full block directly */+ selectedIn = srcPtr;+ srcPtr += dctx->tmpInTarget;++ if (0) /* jump over next block */+ case dstage_storeCBlock:+ { size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize;+ size_t const inputLeft = (size_t)(srcEnd-srcPtr);+ size_t const sizeToCopy = MIN(wantedData, inputLeft);+ memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);+ dctx->tmpInSize += sizeToCopy;+ srcPtr += sizeToCopy;+ if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */+ nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize)+ + (dctx->frameInfo.blockChecksumFlag ? BFSize : 0)+ + BHSize /* next header size */;+ doAnotherStage = 0;+ break;+ }+ selectedIn = dctx->tmpIn;+ }++ /* At this stage, input is large enough to decode a block */+ if (dctx->frameInfo.blockChecksumFlag) {+ dctx->tmpInTarget -= 4;+ assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */+ { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);+ U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION+ if (readBlockCrc != calcBlockCrc)+ return err0r(LZ4F_ERROR_blockChecksum_invalid);+#else+ (void)readBlockCrc;+ (void)calcBlockCrc;+#endif+ } }++ if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {+ const char* dict = (const char*)dctx->dict;+ size_t dictSize = dctx->dictSize;+ int decodedSize;+ if (dict && dictSize > 1 GB) {+ /* the dictSize param is an int, avoid truncation / sign issues */+ dict += dictSize - 64 KB;+ dictSize = 64 KB;+ }+ /* enough capacity in `dst` to decompress directly there */+ decodedSize = LZ4_decompress_safe_usingDict(+ (const char*)selectedIn, (char*)dstPtr,+ (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,+ dict, (int)dictSize);+ if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */+ if (dctx->frameInfo.contentChecksumFlag)+ XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize);+ if (dctx->frameInfo.contentSize)+ dctx->frameRemainingSize -= (size_t)decodedSize;++ /* dictionary management */+ if (dctx->frameInfo.blockMode==LZ4F_blockLinked)+ LZ4F_updateDict(dctx, dstPtr, (size_t)decodedSize, dstStart, 0);++ dstPtr += decodedSize;+ dctx->dStage = dstage_getBlockHeader;+ break;+ }++ /* not enough place into dst : decode into tmpOut */+ /* ensure enough place for tmpOut */+ if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {+ if (dctx->dict == dctx->tmpOutBuffer) {+ if (dctx->dictSize > 128 KB) {+ memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB);+ dctx->dictSize = 64 KB;+ }+ dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize;+ } else { /* dict not within tmp */+ size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);+ dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;+ } }++ /* Decode block */+ { const char* dict = (const char*)dctx->dict;+ size_t dictSize = dctx->dictSize;+ int decodedSize;+ if (dict && dictSize > 1 GB) {+ /* the dictSize param is an int, avoid truncation / sign issues */+ dict += dictSize - 64 KB;+ dictSize = 64 KB;+ }+ decodedSize = LZ4_decompress_safe_usingDict(+ (const char*)selectedIn, (char*)dctx->tmpOut,+ (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,+ dict, (int)dictSize);+ if (decodedSize < 0) /* decompression failed */+ return err0r(LZ4F_ERROR_decompressionFailed);+ if (dctx->frameInfo.contentChecksumFlag)+ XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize);+ if (dctx->frameInfo.contentSize)+ dctx->frameRemainingSize -= (size_t)decodedSize;+ dctx->tmpOutSize = (size_t)decodedSize;+ dctx->tmpOutStart = 0;+ dctx->dStage = dstage_flushOut;+ }+ /* fall-through */++ case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */+ { size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr));+ memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy);++ /* dictionary management */+ if (dctx->frameInfo.blockMode == LZ4F_blockLinked)+ LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1 /*withinTmp*/);++ dctx->tmpOutStart += sizeToCopy;+ dstPtr += sizeToCopy;++ if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */+ dctx->dStage = dstage_getBlockHeader; /* get next block */+ break;+ }+ /* could not flush everything : stop there, just request a block header */+ doAnotherStage = 0;+ nextSrcSizeHint = BHSize;+ break;+ }++ case dstage_getSuffix:+ if (dctx->frameRemainingSize)+ return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */+ if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */+ nextSrcSizeHint = 0;+ LZ4F_resetDecompressionContext(dctx);+ doAnotherStage = 0;+ break;+ }+ if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */+ dctx->tmpInSize = 0;+ dctx->dStage = dstage_storeSuffix;+ } else {+ selectedIn = srcPtr;+ srcPtr += 4;+ }++ if (dctx->dStage == dstage_storeSuffix) /* can be skipped */+ case dstage_storeSuffix:+ { size_t const remainingInput = (size_t)(srcEnd - srcPtr);+ size_t const wantedData = 4 - dctx->tmpInSize;+ size_t const sizeToCopy = MIN(wantedData, remainingInput);+ memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);+ srcPtr += sizeToCopy;+ dctx->tmpInSize += sizeToCopy;+ if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */+ nextSrcSizeHint = 4 - dctx->tmpInSize;+ doAnotherStage=0;+ break;+ }+ selectedIn = dctx->tmpIn;+ } /* if (dctx->dStage == dstage_storeSuffix) */++ /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */+ { U32 const readCRC = LZ4F_readLE32(selectedIn);+ U32 const resultCRC = XXH32_digest(&(dctx->xxh));+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION+ if (readCRC != resultCRC)+ return err0r(LZ4F_ERROR_contentChecksum_invalid);+#else+ (void)readCRC;+ (void)resultCRC;+#endif+ nextSrcSizeHint = 0;+ LZ4F_resetDecompressionContext(dctx);+ doAnotherStage = 0;+ break;+ }++ case dstage_getSFrameSize:+ if ((srcEnd - srcPtr) >= 4) {+ selectedIn = srcPtr;+ srcPtr += 4;+ } else {+ /* not enough input to read cBlockSize field */+ dctx->tmpInSize = 4;+ dctx->tmpInTarget = 8;+ dctx->dStage = dstage_storeSFrameSize;+ }++ if (dctx->dStage == dstage_storeSFrameSize)+ case dstage_storeSFrameSize:+ { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,+ (size_t)(srcEnd - srcPtr) );+ memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);+ srcPtr += sizeToCopy;+ dctx->tmpInSize += sizeToCopy;+ if (dctx->tmpInSize < dctx->tmpInTarget) {+ /* not enough input to get full sBlockSize; wait for more */+ nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize;+ doAnotherStage = 0;+ break;+ }+ selectedIn = dctx->header + 4;+ } /* if (dctx->dStage == dstage_storeSFrameSize) */++ /* case dstage_decodeSFrameSize: */ /* no direct entry */+ { size_t const SFrameSize = LZ4F_readLE32(selectedIn);+ dctx->frameInfo.contentSize = SFrameSize;+ dctx->tmpInTarget = SFrameSize;+ dctx->dStage = dstage_skipSkippable;+ break;+ }++ case dstage_skipSkippable:+ { size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr));+ srcPtr += skipSize;+ dctx->tmpInTarget -= skipSize;+ doAnotherStage = 0;+ nextSrcSizeHint = dctx->tmpInTarget;+ if (nextSrcSizeHint) break; /* still more to skip */+ /* frame fully skipped : prepare context for a new frame */+ LZ4F_resetDecompressionContext(dctx);+ break;+ }+ } /* switch (dctx->dStage) */+ } /* while (doAnotherStage) */++ /* preserve history within tmp whenever necessary */+ LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2);+ if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */+ && (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */+ && (!decompressOptionsPtr->stableDst) /* cannot rely on dst data to remain there for next call */+ && ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */+ {+ if (dctx->dStage == dstage_flushOut) {+ size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer);+ size_t copySize = 64 KB - dctx->tmpOutSize;+ const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;+ if (dctx->tmpOutSize > 64 KB) copySize = 0;+ if (copySize > preserveSize) copySize = preserveSize;++ if (copySize > 0)+ memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);++ dctx->dict = dctx->tmpOutBuffer;+ dctx->dictSize = preserveSize + dctx->tmpOutStart;+ } else {+ const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize;+ size_t const newDictSize = MIN(dctx->dictSize, 64 KB);++ if (newDictSize > 0)+ memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);++ dctx->dict = dctx->tmpOutBuffer;+ dctx->dictSize = newDictSize;+ dctx->tmpOut = dctx->tmpOutBuffer + newDictSize;+ }+ }++ *srcSizePtr = (size_t)(srcPtr - srcStart);+ *dstSizePtr = (size_t)(dstPtr - dstStart);+ return nextSrcSizeHint;+}++/*! LZ4F_decompress_usingDict() :+ * Same as LZ4F_decompress(), using a predefined dictionary.+ * Dictionary is used "in place", without any preprocessing.+ * It must remain accessible throughout the entire frame decoding.+ */+size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx,+ void* dstBuffer, size_t* dstSizePtr,+ const void* srcBuffer, size_t* srcSizePtr,+ const void* dict, size_t dictSize,+ const LZ4F_decompressOptions_t* decompressOptionsPtr)+{+ if (dctx->dStage <= dstage_init) {+ dctx->dict = (const BYTE*)dict;+ dctx->dictSize = dictSize;+ }+ return LZ4F_decompress(dctx, dstBuffer, dstSizePtr,+ srcBuffer, srcSizePtr,+ decompressOptionsPtr);+}
+ lz4/lib/lz4frame.h view
@@ -0,0 +1,615 @@+/*+ LZ4 auto-framing library+ Header File+ Copyright (C) 2011-2017, Yann Collet.+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)++ Redistribution and use in source and binary forms, with or without+ modification, are permitted provided that the following conditions are+ met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.+ * Redistributions in binary form must reproduce the above+ copyright notice, this list of conditions and the following disclaimer+ in the documentation and/or other materials provided with the+ distribution.++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.++ You can contact the author at :+ - LZ4 source repository : https://github.com/lz4/lz4+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c+*/++/* LZ4F is a stand-alone API able to create and decode LZ4 frames+ * conformant with specification v1.6.1 in doc/lz4_Frame_format.md .+ * Generated frames are compatible with `lz4` CLI.+ *+ * LZ4F also offers streaming capabilities.+ *+ * lz4.h is not required when using lz4frame.h,+ * except to extract common constant such as LZ4_VERSION_NUMBER.+ * */++#ifndef LZ4F_H_09782039843+#define LZ4F_H_09782039843++#if defined (__cplusplus)+extern "C" {+#endif++/* --- Dependency --- */+#include <stddef.h> /* size_t */+++/**+ Introduction++ lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md).+ lz4frame.h provides frame compression functions that take care+ of encoding standard metadata alongside LZ4-compressed blocks.+*/++/*-***************************************************************+ * Compiler specifics+ *****************************************************************/+/* LZ4_DLL_EXPORT :+ * Enable exporting of functions when building a Windows DLL+ * LZ4FLIB_API :+ * Control library symbols visibility.+ */+#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)+# define LZ4FLIB_API __declspec(dllexport)+#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)+# define LZ4FLIB_API __declspec(dllimport)+#elif defined(__GNUC__) && (__GNUC__ >= 4)+# define LZ4FLIB_API __attribute__ ((__visibility__ ("default")))+#else+# define LZ4FLIB_API+#endif++#ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS+# define LZ4F_DEPRECATE(x) x+#else+# if defined(_MSC_VER)+# define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */+# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6))+# define LZ4F_DEPRECATE(x) x __attribute__((deprecated))+# else+# define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */+# endif+#endif+++/*-************************************+ * Error management+ **************************************/+typedef size_t LZ4F_errorCode_t;++LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells when a function result is an error code */+LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; for debugging */+++/*-************************************+ * Frame compression types+ **************************************/+/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */+#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS+# define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x+#else+# define LZ4F_OBSOLETE_ENUM(x)+#endif++/* The larger the block size, the (slightly) better the compression ratio,+ * though there are diminishing returns.+ * Larger blocks also increase memory usage on both compression and decompression sides. */+typedef enum {+ LZ4F_default=0,+ LZ4F_max64KB=4,+ LZ4F_max256KB=5,+ LZ4F_max1MB=6,+ LZ4F_max4MB=7+ LZ4F_OBSOLETE_ENUM(max64KB)+ LZ4F_OBSOLETE_ENUM(max256KB)+ LZ4F_OBSOLETE_ENUM(max1MB)+ LZ4F_OBSOLETE_ENUM(max4MB)+} LZ4F_blockSizeID_t;++/* Linked blocks sharply reduce inefficiencies when using small blocks,+ * they compress better.+ * However, some LZ4 decoders are only compatible with independent blocks */+typedef enum {+ LZ4F_blockLinked=0,+ LZ4F_blockIndependent+ LZ4F_OBSOLETE_ENUM(blockLinked)+ LZ4F_OBSOLETE_ENUM(blockIndependent)+} LZ4F_blockMode_t;++typedef enum {+ LZ4F_noContentChecksum=0,+ LZ4F_contentChecksumEnabled+ LZ4F_OBSOLETE_ENUM(noContentChecksum)+ LZ4F_OBSOLETE_ENUM(contentChecksumEnabled)+} LZ4F_contentChecksum_t;++typedef enum {+ LZ4F_noBlockChecksum=0,+ LZ4F_blockChecksumEnabled+} LZ4F_blockChecksum_t;++typedef enum {+ LZ4F_frame=0,+ LZ4F_skippableFrame+ LZ4F_OBSOLETE_ENUM(skippableFrame)+} LZ4F_frameType_t;++#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS+typedef LZ4F_blockSizeID_t blockSizeID_t;+typedef LZ4F_blockMode_t blockMode_t;+typedef LZ4F_frameType_t frameType_t;+typedef LZ4F_contentChecksum_t contentChecksum_t;+#endif++/*! LZ4F_frameInfo_t :+ * makes it possible to set or read frame parameters.+ * Structure must be first init to 0, using memset() or LZ4F_INIT_FRAMEINFO,+ * setting all parameters to default.+ * It's then possible to update selectively some parameters */+typedef struct {+ LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB; 0 == default */+ LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */+ LZ4F_contentChecksum_t contentChecksumFlag; /* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */+ LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */+ unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */+ unsigned dictID; /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */+ LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */+} LZ4F_frameInfo_t;++#define LZ4F_INIT_FRAMEINFO { LZ4F_default, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0ULL, 0U, LZ4F_noBlockChecksum } /* v1.8.3+ */++/*! LZ4F_preferences_t :+ * makes it possible to supply advanced compression instructions to streaming interface.+ * Structure must be first init to 0, using memset() or LZ4F_INIT_PREFERENCES,+ * setting all parameters to default.+ * All reserved fields must be set to zero. */+typedef struct {+ LZ4F_frameInfo_t frameInfo;+ int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */+ unsigned autoFlush; /* 1: always flush; reduces usage of internal buffers */+ unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4HC_CLEVEL_OPT_MIN) */ /* v1.8.2+ */+ unsigned reserved[3]; /* must be zero for forward compatibility */+} LZ4F_preferences_t;++#define LZ4F_INIT_PREFERENCES { LZ4F_INIT_FRAMEINFO, 0, 0u, 0u, { 0u, 0u, 0u } } /* v1.8.3+ */+++/*-*********************************+* Simple compression function+***********************************/++LZ4FLIB_API int LZ4F_compressionLevel_max(void); /* v1.8.0+ */++/*! LZ4F_compressFrameBound() :+ * Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.+ * `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.+ * Note : this result is only usable with LZ4F_compressFrame().+ * It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.+ */+LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);++/*! LZ4F_compressFrame() :+ * Compress an entire srcBuffer into a valid LZ4 frame.+ * dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).+ * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.+ * @return : number of bytes written into dstBuffer.+ * or an error code if it fails (can be tested using LZ4F_isError())+ */+LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,+ const void* srcBuffer, size_t srcSize,+ const LZ4F_preferences_t* preferencesPtr);+++/*-***********************************+* Advanced compression functions+*************************************/+typedef struct LZ4F_cctx_s LZ4F_cctx; /* incomplete type */+typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with previous API version */++typedef struct {+ unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */+ unsigned reserved[3];+} LZ4F_compressOptions_t;++/*--- Resource Management ---*/++#define LZ4F_VERSION 100 /* This number can be used to check for an incompatible API breaking change */+LZ4FLIB_API unsigned LZ4F_getVersion(void);++/*! LZ4F_createCompressionContext() :+ * The first thing to do is to create a compressionContext object, which will be used in all compression operations.+ * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.+ * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.+ * The function will provide a pointer to a fully allocated LZ4F_cctx object.+ * If @return != zero, there was an error during context creation.+ * Object can release its memory using LZ4F_freeCompressionContext();+ */+LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);+LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);+++/*---- Compression ----*/++#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */+#define LZ4F_HEADER_SIZE_MAX 19++/* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */+#define LZ4F_BLOCK_HEADER_SIZE 4++/* Size in bytes of a block checksum footer in little-endian format. */+#define LZ4F_BLOCK_CHECKSUM_SIZE 4++/* Size in bytes of the content checksum. */+#define LZ4F_CONTENT_CHECKSUM_SIZE 4++/*! LZ4F_compressBegin() :+ * will write the frame header into dstBuffer.+ * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.+ * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.+ * @return : number of bytes written into dstBuffer for the header+ * or an error code (which can be tested using LZ4F_isError())+ */+LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,+ void* dstBuffer, size_t dstCapacity,+ const LZ4F_preferences_t* prefsPtr);++/*! LZ4F_compressBound() :+ * Provides minimum dstCapacity required to guarantee success of+ * LZ4F_compressUpdate(), given a srcSize and preferences, for a worst case scenario.+ * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() instead.+ * Note that the result is only valid for a single invocation of LZ4F_compressUpdate().+ * When invoking LZ4F_compressUpdate() multiple times,+ * if the output buffer is gradually filled up instead of emptied and re-used from its start,+ * one must check if there is enough remaining capacity before each invocation, using LZ4F_compressBound().+ * @return is always the same for a srcSize and prefsPtr.+ * prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.+ * tech details :+ * @return includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes.+ * It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd().+ * @return doesn't include frame header, as it was already generated by LZ4F_compressBegin().+ */+LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);++/*! LZ4F_compressUpdate() :+ * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.+ * Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.+ * This value is provided by LZ4F_compressBound().+ * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).+ * LZ4F_compressUpdate() doesn't guarantee error recovery.+ * When an error occurs, compression context must be freed or resized.+ * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.+ * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).+ * or an error code if it fails (which can be tested using LZ4F_isError())+ */+LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,+ void* dstBuffer, size_t dstCapacity,+ const void* srcBuffer, size_t srcSize,+ const LZ4F_compressOptions_t* cOptPtr);++/*! LZ4F_flush() :+ * When data must be generated and sent immediately, without waiting for a block to be completely filled,+ * it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.+ * `dstCapacity` must be large enough to ensure the operation will be successful.+ * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.+ * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)+ * or an error code if it fails (which can be tested using LZ4F_isError())+ * Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr).+ */+LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx,+ void* dstBuffer, size_t dstCapacity,+ const LZ4F_compressOptions_t* cOptPtr);++/*! LZ4F_compressEnd() :+ * To properly finish an LZ4 frame, invoke LZ4F_compressEnd().+ * It will flush whatever data remained within `cctx` (like LZ4_flush())+ * and properly finalize the frame, with an endMark and a checksum.+ * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.+ * @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark),+ * or an error code if it fails (which can be tested using LZ4F_isError())+ * Note : LZ4F_compressEnd() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr).+ * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.+ */+LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx,+ void* dstBuffer, size_t dstCapacity,+ const LZ4F_compressOptions_t* cOptPtr);+++/*-*********************************+* Decompression functions+***********************************/+typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */+typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */++typedef struct {+ unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */+ unsigned reserved[3]; /* must be set to zero for forward compatibility */+} LZ4F_decompressOptions_t;+++/* Resource management */++/*! LZ4F_createDecompressionContext() :+ * Create an LZ4F_dctx object, to track all decompression operations.+ * The version provided MUST be LZ4F_VERSION.+ * The function provides a pointer to an allocated and initialized LZ4F_dctx object.+ * The result is an errorCode, which can be tested using LZ4F_isError().+ * dctx memory can be released using LZ4F_freeDecompressionContext();+ * Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.+ * That is, it should be == 0 if decompression has been completed fully and correctly.+ */+LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);+LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);+++/*-***********************************+* Streaming decompression functions+*************************************/++#define LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH 5++/*! LZ4F_headerSize() : v1.9.0++ * Provide the header size of a frame starting at `src`.+ * `srcSize` must be >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH,+ * which is enough to decode the header length.+ * @return : size of frame header+ * or an error code, which can be tested using LZ4F_isError()+ * note : Frame header size is variable, but is guaranteed to be+ * >= LZ4F_HEADER_SIZE_MIN bytes, and <= LZ4F_HEADER_SIZE_MAX bytes.+ */+size_t LZ4F_headerSize(const void* src, size_t srcSize);++/*! LZ4F_getFrameInfo() :+ * This function extracts frame parameters (max blockSize, dictID, etc.).+ * Its usage is optional: user can call LZ4F_decompress() directly.+ *+ * Extracted information will fill an existing LZ4F_frameInfo_t structure.+ * This can be useful for allocation and dictionary identification purposes.+ *+ * LZ4F_getFrameInfo() can work in the following situations :+ *+ * 1) At the beginning of a new frame, before any invocation of LZ4F_decompress().+ * It will decode header from `srcBuffer`,+ * consuming the header and starting the decoding process.+ *+ * Input size must be large enough to contain the full frame header.+ * Frame header size can be known beforehand by LZ4F_headerSize().+ * Frame header size is variable, but is guaranteed to be >= LZ4F_HEADER_SIZE_MIN bytes,+ * and not more than <= LZ4F_HEADER_SIZE_MAX bytes.+ * Hence, blindly providing LZ4F_HEADER_SIZE_MAX bytes or more will always work.+ * It's allowed to provide more input data than the header size,+ * LZ4F_getFrameInfo() will only consume the header.+ *+ * If input size is not large enough,+ * aka if it's smaller than header size,+ * function will fail and return an error code.+ *+ * 2) After decoding has been started,+ * it's possible to invoke LZ4F_getFrameInfo() anytime+ * to extract already decoded frame parameters stored within dctx.+ *+ * Note that, if decoding has barely started,+ * and not yet read enough information to decode the header,+ * LZ4F_getFrameInfo() will fail.+ *+ * The number of bytes consumed from srcBuffer will be updated in *srcSizePtr (necessarily <= original value).+ * LZ4F_getFrameInfo() only consumes bytes when decoding has not yet started,+ * and when decoding the header has been successful.+ * Decompression must then resume from (srcBuffer + *srcSizePtr).+ *+ * @return : a hint about how many srcSize bytes LZ4F_decompress() expects for next call,+ * or an error code which can be tested using LZ4F_isError().+ * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.+ * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.+ */+LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,+ LZ4F_frameInfo_t* frameInfoPtr,+ const void* srcBuffer, size_t* srcSizePtr);++/*! LZ4F_decompress() :+ * Call this function repetitively to regenerate compressed data from `srcBuffer`.+ * The function will read up to *srcSizePtr bytes from srcBuffer,+ * and decompress data into dstBuffer, of capacity *dstSizePtr.+ *+ * The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).+ * The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).+ *+ * The function does not necessarily read all input bytes, so always check value in *srcSizePtr.+ * Unconsumed source data must be presented again in subsequent invocations.+ *+ * `dstBuffer` can freely change between each consecutive function invocation.+ * `dstBuffer` content will be overwritten.+ *+ * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.+ * Schematically, it's the size of the current (or remaining) compressed block + header of next block.+ * Respecting the hint provides some small speed benefit, because it skips intermediate buffers.+ * This is just a hint though, it's always possible to provide any srcSize.+ *+ * When a frame is fully decoded, @return will be 0 (no more data expected).+ * When provided with more bytes than necessary to decode a frame,+ * LZ4F_decompress() will stop reading exactly at end of current frame, and @return 0.+ *+ * If decompression failed, @return is an error code, which can be tested using LZ4F_isError().+ * After a decompression error, the `dctx` context is not resumable.+ * Use LZ4F_resetDecompressionContext() to return to clean state.+ *+ * After a frame is fully decoded, dctx can be used again to decompress another frame.+ */+LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,+ void* dstBuffer, size_t* dstSizePtr,+ const void* srcBuffer, size_t* srcSizePtr,+ const LZ4F_decompressOptions_t* dOptPtr);+++/*! LZ4F_resetDecompressionContext() : added in v1.8.0+ * In case of an error, the context is left in "undefined" state.+ * In which case, it's necessary to reset it, before re-using it.+ * This method can also be used to abruptly stop any unfinished decompression,+ * and start a new one using same context resources. */+LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */++++#if defined (__cplusplus)+}+#endif++#endif /* LZ4F_H_09782039843 */++#if defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843)+#define LZ4F_H_STATIC_09782039843++#if defined (__cplusplus)+extern "C" {+#endif++/* These declarations are not stable and may change in the future.+ * They are therefore only safe to depend on+ * when the caller is statically linked against the library.+ * To access their declarations, define LZ4F_STATIC_LINKING_ONLY.+ *+ * By default, these symbols aren't published into shared/dynamic libraries.+ * You can override this behavior and force them to be published+ * by defining LZ4F_PUBLISH_STATIC_FUNCTIONS.+ * Use at your own risk.+ */+#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS+#define LZ4FLIB_STATIC_API LZ4FLIB_API+#else+#define LZ4FLIB_STATIC_API+#endif+++/* --- Error List --- */+#define LZ4F_LIST_ERRORS(ITEM) \+ ITEM(OK_NoError) \+ ITEM(ERROR_GENERIC) \+ ITEM(ERROR_maxBlockSize_invalid) \+ ITEM(ERROR_blockMode_invalid) \+ ITEM(ERROR_contentChecksumFlag_invalid) \+ ITEM(ERROR_compressionLevel_invalid) \+ ITEM(ERROR_headerVersion_wrong) \+ ITEM(ERROR_blockChecksum_invalid) \+ ITEM(ERROR_reservedFlag_set) \+ ITEM(ERROR_allocation_failed) \+ ITEM(ERROR_srcSize_tooLarge) \+ ITEM(ERROR_dstMaxSize_tooSmall) \+ ITEM(ERROR_frameHeader_incomplete) \+ ITEM(ERROR_frameType_unknown) \+ ITEM(ERROR_frameSize_wrong) \+ ITEM(ERROR_srcPtr_wrong) \+ ITEM(ERROR_decompressionFailed) \+ ITEM(ERROR_headerChecksum_invalid) \+ ITEM(ERROR_contentChecksum_invalid) \+ ITEM(ERROR_frameDecoding_alreadyStarted) \+ ITEM(ERROR_maxCode)++#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,++/* enum list is exposed, to handle specific errors */+typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM)+ _LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes;++LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);++LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned);++/**********************************+ * Bulk processing dictionary API+ *********************************/++/* A Dictionary is useful for the compression of small messages (KB range).+ * It dramatically improves compression efficiency.+ *+ * LZ4 can ingest any input as dictionary, though only the last 64 KB are useful.+ * Best results are generally achieved by using Zstandard's Dictionary Builder+ * to generate a high-quality dictionary from a set of samples.+ *+ * Loading a dictionary has a cost, since it involves construction of tables.+ * The Bulk processing dictionary API makes it possible to share this cost+ * over an arbitrary number of compression jobs, even concurrently,+ * markedly improving compression latency for these cases.+ *+ * The same dictionary will have to be used on the decompression side+ * for decoding to be successful.+ * To help identify the correct dictionary at decoding stage,+ * the frame header allows optional embedding of a dictID field.+ */+typedef struct LZ4F_CDict_s LZ4F_CDict;++/*! LZ4_createCDict() :+ * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it just once.+ * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.+ * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.+ * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */+LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);+LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict);+++/*! LZ4_compressFrame_usingCDict() :+ * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.+ * cctx must point to a context created by LZ4F_createCompressionContext().+ * If cdict==NULL, compress without a dictionary.+ * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).+ * If this condition is not respected, function will fail (@return an errorCode).+ * The LZ4F_preferences_t structure is optional : you may provide NULL as argument,+ * but it's not recommended, as it's the only way to provide dictID in the frame header.+ * @return : number of bytes written into dstBuffer.+ * or an error code if it fails (can be tested using LZ4F_isError()) */+LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(+ LZ4F_cctx* cctx,+ void* dst, size_t dstCapacity,+ const void* src, size_t srcSize,+ const LZ4F_CDict* cdict,+ const LZ4F_preferences_t* preferencesPtr);+++/*! LZ4F_compressBegin_usingCDict() :+ * Inits streaming dictionary compression, and writes the frame header into dstBuffer.+ * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.+ * `prefsPtr` is optional : you may provide NULL as argument,+ * however, it's the only way to provide dictID in the frame header.+ * @return : number of bytes written into dstBuffer for the header,+ * or an error code (which can be tested using LZ4F_isError()) */+LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(+ LZ4F_cctx* cctx,+ void* dstBuffer, size_t dstCapacity,+ const LZ4F_CDict* cdict,+ const LZ4F_preferences_t* prefsPtr);+++/*! LZ4F_decompress_usingDict() :+ * Same as LZ4F_decompress(), using a predefined dictionary.+ * Dictionary is used "in place", without any preprocessing.+ * It must remain accessible throughout the entire frame decoding. */+LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(+ LZ4F_dctx* dctxPtr,+ void* dstBuffer, size_t* dstSizePtr,+ const void* srcBuffer, size_t* srcSizePtr,+ const void* dict, size_t dictSize,+ const LZ4F_decompressOptions_t* decompressOptionsPtr);++#if defined (__cplusplus)+}+#endif++#endif /* defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) */
+ lz4/lib/lz4frame_static.h view
@@ -0,0 +1,47 @@+/*+ LZ4 auto-framing library+ Header File for static linking only+ Copyright (C) 2011-2016, Yann Collet.++ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)++ Redistribution and use in source and binary forms, with or without+ modification, are permitted provided that the following conditions are+ met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.+ * Redistributions in binary form must reproduce the above+ copyright notice, this list of conditions and the following disclaimer+ in the documentation and/or other materials provided with the+ distribution.++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.++ You can contact the author at :+ - LZ4 source repository : https://github.com/lz4/lz4+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c+*/++#ifndef LZ4FRAME_STATIC_H_0398209384+#define LZ4FRAME_STATIC_H_0398209384++/* The declarations that formerly were made here have been merged into+ * lz4frame.h, protected by the LZ4F_STATIC_LINKING_ONLY macro. Going forward,+ * it is recommended to simply include that header directly.+ */++#define LZ4F_STATIC_LINKING_ONLY+#include "lz4frame.h"++#endif /* LZ4FRAME_STATIC_H_0398209384 */
+ lz4/lib/lz4hc.c view
@@ -0,0 +1,1538 @@+/*+ LZ4 HC - High Compression Mode of LZ4+ Copyright (C) 2011-2017, Yann Collet.++ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)++ Redistribution and use in source and binary forms, with or without+ modification, are permitted provided that the following conditions are+ met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.+ * Redistributions in binary form must reproduce the above+ copyright notice, this list of conditions and the following disclaimer+ in the documentation and/or other materials provided with the+ distribution.++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.++ You can contact the author at :+ - LZ4 source repository : https://github.com/lz4/lz4+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c+*/+/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */+++/* *************************************+* Tuning Parameter+***************************************/++/*! HEAPMODE :+ * Select how default compression function will allocate workplace memory,+ * in stack (0:fastest), or in heap (1:requires malloc()).+ * Since workplace is rather large, heap mode is recommended.+ */+#ifndef LZ4HC_HEAPMODE+# define LZ4HC_HEAPMODE 1+#endif+++/*=== Dependency ===*/+#define LZ4_HC_STATIC_LINKING_ONLY+#include "lz4hc.h"+++/*=== Common LZ4 definitions ===*/+#if defined(__GNUC__)+# pragma GCC diagnostic ignored "-Wunused-function"+#endif+#if defined (__clang__)+# pragma clang diagnostic ignored "-Wunused-function"+#endif++/*=== Enums ===*/+typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive;+++#define LZ4_COMMONDEFS_ONLY+#ifndef LZ4_SRC_INCLUDED+#include "lz4.c" /* LZ4_count, constants, mem */+#endif++/*=== Constants ===*/+#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)+#define LZ4_OPT_NUM (1<<12)+++/*=== Macros ===*/+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )+#define MAX(a,b) ( (a) > (b) ? (a) : (b) )+#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))+#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */+#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */+/* Make fields passed to, and updated by LZ4HC_encodeSequence explicit */+#define UPDATABLE(ip, op, anchor) &ip, &op, &anchor++static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }+++/**************************************+* HC Compression+**************************************/+static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)+{+ MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));+ MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));+}++static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start)+{+ uptrval startingOffset = (uptrval)(hc4->end - hc4->base);+ if (startingOffset > 1 GB) {+ LZ4HC_clearTables(hc4);+ startingOffset = 0;+ }+ startingOffset += 64 KB;+ hc4->nextToUpdate = (U32) startingOffset;+ hc4->base = start - startingOffset;+ hc4->end = start;+ hc4->dictBase = start - startingOffset;+ hc4->dictLimit = (U32) startingOffset;+ hc4->lowLimit = (U32) startingOffset;+}+++/* Update chains up to ip (excluded) */+LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)+{+ U16* const chainTable = hc4->chainTable;+ U32* const hashTable = hc4->hashTable;+ const BYTE* const base = hc4->base;+ U32 const target = (U32)(ip - base);+ U32 idx = hc4->nextToUpdate;++ while (idx < target) {+ U32 const h = LZ4HC_hashPtr(base+idx);+ size_t delta = idx - hashTable[h];+ if (delta>LZ4_DISTANCE_MAX) delta = LZ4_DISTANCE_MAX;+ DELTANEXTU16(chainTable, idx) = (U16)delta;+ hashTable[h] = idx;+ idx++;+ }++ hc4->nextToUpdate = target;+}++/** LZ4HC_countBack() :+ * @return : negative value, nb of common bytes before ip/match */+LZ4_FORCE_INLINE+int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,+ const BYTE* const iMin, const BYTE* const mMin)+{+ int back = 0;+ int const min = (int)MAX(iMin - ip, mMin - match);+ assert(min <= 0);+ assert(ip >= iMin); assert((size_t)(ip-iMin) < (1U<<31));+ assert(match >= mMin); assert((size_t)(match - mMin) < (1U<<31));+ while ( (back > min)+ && (ip[back-1] == match[back-1]) )+ back--;+ return back;+}++#if defined(_MSC_VER)+# define LZ4HC_rotl32(x,r) _rotl(x,r)+#else+# define LZ4HC_rotl32(x,r) ((x << r) | (x >> (32 - r)))+#endif+++static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern)+{+ size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3;+ if (bitsToRotate == 0)+ return pattern;+ return LZ4HC_rotl32(pattern, (int)bitsToRotate);+}++/* LZ4HC_countPattern() :+ * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */+static unsigned+LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32)+{+ const BYTE* const iStart = ip;+ reg_t const pattern = (sizeof(pattern)==8) ? (reg_t)pattern32 + (((reg_t)pattern32) << 32) : pattern32;++ while (likely(ip < iEnd-(sizeof(pattern)-1))) {+ reg_t const diff = LZ4_read_ARCH(ip) ^ pattern;+ if (!diff) { ip+=sizeof(pattern); continue; }+ ip += LZ4_NbCommonBytes(diff);+ return (unsigned)(ip - iStart);+ }++ if (LZ4_isLittleEndian()) {+ reg_t patternByte = pattern;+ while ((ip<iEnd) && (*ip == (BYTE)patternByte)) {+ ip++; patternByte >>= 8;+ }+ } else { /* big endian */+ U32 bitOffset = (sizeof(pattern)*8) - 8;+ while (ip < iEnd) {+ BYTE const byte = (BYTE)(pattern >> bitOffset);+ if (*ip != byte) break;+ ip ++; bitOffset -= 8;+ }+ }++ return (unsigned)(ip - iStart);+}++/* LZ4HC_reverseCountPattern() :+ * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!)+ * read using natural platform endianess */+static unsigned+LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)+{+ const BYTE* const iStart = ip;++ while (likely(ip >= iLow+4)) {+ if (LZ4_read32(ip-4) != pattern) break;+ ip -= 4;+ }+ { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianess */+ while (likely(ip>iLow)) {+ if (ip[-1] != *bytePtr) break;+ ip--; bytePtr--;+ } }+ return (unsigned)(iStart - ip);+}++/* LZ4HC_protectDictEnd() :+ * Checks if the match is in the last 3 bytes of the dictionary, so reading the+ * 4 byte MINMATCH would overflow.+ * @returns true if the match index is okay.+ */+static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex)+{+ return ((U32)((dictLimit - 1) - matchIndex) >= 3);+}++typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e;+typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e;++LZ4_FORCE_INLINE int+LZ4HC_InsertAndGetWiderMatch (+ LZ4HC_CCtx_internal* hc4,+ const BYTE* const ip,+ const BYTE* const iLowLimit,+ const BYTE* const iHighLimit,+ int longest,+ const BYTE** matchpos,+ const BYTE** startpos,+ const int maxNbAttempts,+ const int patternAnalysis,+ const int chainSwap,+ const dictCtx_directive dict,+ const HCfavor_e favorDecSpeed)+{+ U16* const chainTable = hc4->chainTable;+ U32* const HashTable = hc4->hashTable;+ const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx;+ const BYTE* const base = hc4->base;+ const U32 dictLimit = hc4->dictLimit;+ const BYTE* const lowPrefixPtr = base + dictLimit;+ const U32 ipIndex = (U32)(ip - base);+ const U32 lowestMatchIndex = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX;+ const BYTE* const dictBase = hc4->dictBase;+ int const lookBackLength = (int)(ip-iLowLimit);+ int nbAttempts = maxNbAttempts;+ U32 matchChainPos = 0;+ U32 const pattern = LZ4_read32(ip);+ U32 matchIndex;+ repeat_state_e repeat = rep_untested;+ size_t srcPatternLength = 0;++ DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch");+ /* First Match */+ LZ4HC_Insert(hc4, ip);+ matchIndex = HashTable[LZ4HC_hashPtr(ip)];+ DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)",+ matchIndex, lowestMatchIndex);++ while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {+ int matchLength=0;+ nbAttempts--;+ assert(matchIndex < ipIndex);+ if (favorDecSpeed && (ipIndex - matchIndex < 8)) {+ /* do nothing */+ } else if (matchIndex >= dictLimit) { /* within current Prefix */+ const BYTE* const matchPtr = base + matchIndex;+ assert(matchPtr >= lowPrefixPtr);+ assert(matchPtr < ip);+ assert(longest >= 1);+ if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) {+ if (LZ4_read32(matchPtr) == pattern) {+ int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;+ matchLength = MINMATCH + (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);+ matchLength -= back;+ if (matchLength > longest) {+ longest = matchLength;+ *matchpos = matchPtr + back;+ *startpos = ip + back;+ } } }+ } else { /* lowestMatchIndex <= matchIndex < dictLimit */+ const BYTE* const matchPtr = dictBase + matchIndex;+ if (LZ4_read32(matchPtr) == pattern) {+ const BYTE* const dictStart = dictBase + hc4->lowLimit;+ int back = 0;+ const BYTE* vLimit = ip + (dictLimit - matchIndex);+ if (vLimit > iHighLimit) vLimit = iHighLimit;+ matchLength = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;+ if ((ip+matchLength == vLimit) && (vLimit < iHighLimit))+ matchLength += LZ4_count(ip+matchLength, lowPrefixPtr, iHighLimit);+ back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0;+ matchLength -= back;+ if (matchLength > longest) {+ longest = matchLength;+ *matchpos = base + matchIndex + back; /* virtual pos, relative to ip, to retrieve offset */+ *startpos = ip + back;+ } } }++ if (chainSwap && matchLength==longest) { /* better match => select a better chain */+ assert(lookBackLength==0); /* search forward only */+ if (matchIndex + (U32)longest <= ipIndex) {+ int const kTrigger = 4;+ U32 distanceToNextMatch = 1;+ int const end = longest - MINMATCH + 1;+ int step = 1;+ int accel = 1 << kTrigger;+ int pos;+ for (pos = 0; pos < end; pos += step) {+ U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos);+ step = (accel++ >> kTrigger);+ if (candidateDist > distanceToNextMatch) {+ distanceToNextMatch = candidateDist;+ matchChainPos = (U32)pos;+ accel = 1 << kTrigger;+ }+ }+ if (distanceToNextMatch > 1) {+ if (distanceToNextMatch > matchIndex) break; /* avoid overflow */+ matchIndex -= distanceToNextMatch;+ continue;+ } } }++ { U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex);+ if (patternAnalysis && distNextMatch==1 && matchChainPos==0) {+ U32 const matchCandidateIdx = matchIndex-1;+ /* may be a repeated pattern */+ if (repeat == rep_untested) {+ if ( ((pattern & 0xFFFF) == (pattern >> 16))+ & ((pattern & 0xFF) == (pattern >> 24)) ) {+ repeat = rep_confirmed;+ srcPatternLength = LZ4HC_countPattern(ip+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);+ } else {+ repeat = rep_not;+ } }+ if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex)+ && LZ4HC_protectDictEnd(dictLimit, matchCandidateIdx) ) {+ const int extDict = matchCandidateIdx < dictLimit;+ const BYTE* const matchPtr = (extDict ? dictBase : base) + matchCandidateIdx;+ if (LZ4_read32(matchPtr) == pattern) { /* good candidate */+ const BYTE* const dictStart = dictBase + hc4->lowLimit;+ const BYTE* const iLimit = extDict ? dictBase + dictLimit : iHighLimit;+ size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern);+ if (extDict && matchPtr + forwardPatternLength == iLimit) {+ U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern);+ forwardPatternLength += LZ4HC_countPattern(lowPrefixPtr, iHighLimit, rotatedPattern);+ }+ { const BYTE* const lowestMatchPtr = extDict ? dictStart : lowPrefixPtr;+ size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern);+ size_t currentSegmentLength;+ if (!extDict && matchPtr - backLength == lowPrefixPtr && hc4->lowLimit < dictLimit) {+ U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern);+ backLength += LZ4HC_reverseCountPattern(dictBase + dictLimit, dictStart, rotatedPattern);+ }+ /* Limit backLength not go further than lowestMatchIndex */+ backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex);+ assert(matchCandidateIdx - backLength >= lowestMatchIndex);+ currentSegmentLength = backLength + forwardPatternLength;+ /* Adjust to end of pattern if the source pattern fits, otherwise the beginning of the pattern */+ if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */+ && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */+ U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */+ if (LZ4HC_protectDictEnd(dictLimit, newMatchIndex))+ matchIndex = newMatchIndex;+ else {+ /* Can only happen if started in the prefix */+ assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict);+ matchIndex = dictLimit;+ }+ } else {+ U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */+ if (!LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) {+ assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict);+ matchIndex = dictLimit;+ } else {+ matchIndex = newMatchIndex;+ if (lookBackLength==0) { /* no back possible */+ size_t const maxML = MIN(currentSegmentLength, srcPatternLength);+ if ((size_t)longest < maxML) {+ assert(base + matchIndex < ip);+ if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break;+ assert(maxML < 2 GB);+ longest = (int)maxML;+ *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */+ *startpos = ip;+ }+ { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex);+ if (distToNextPattern > matchIndex) break; /* avoid overflow */+ matchIndex -= distToNextPattern;+ } } } } }+ continue;+ } }+ } } /* PA optimization */++ /* follow current chain */+ matchIndex -= DELTANEXTU16(chainTable, matchIndex + matchChainPos);++ } /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */++ if ( dict == usingDictCtxHc+ && nbAttempts+ && ipIndex - lowestMatchIndex < LZ4_DISTANCE_MAX) {+ size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->base);+ U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];+ assert(dictEndOffset <= 1 GB);+ matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset;+ while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) {+ const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;++ if (LZ4_read32(matchPtr) == pattern) {+ int mlt;+ int back = 0;+ const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex);+ if (vLimit > iHighLimit) vLimit = iHighLimit;+ mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;+ back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;+ mlt -= back;+ if (mlt > longest) {+ longest = mlt;+ *matchpos = base + matchIndex + back;+ *startpos = ip + back;+ } }++ { U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex);+ dictMatchIndex -= nextOffset;+ matchIndex -= nextOffset;+ } } }++ return longest;+}++LZ4_FORCE_INLINE+int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */+ const BYTE* const ip, const BYTE* const iLimit,+ const BYTE** matchpos,+ const int maxNbAttempts,+ const int patternAnalysis,+ const dictCtx_directive dict)+{+ const BYTE* uselessPtr = ip;+ /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),+ * but this won't be the case here, as we define iLowLimit==ip,+ * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */+ return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio);+}++/* LZ4HC_encodeSequence() :+ * @return : 0 if ok,+ * 1 if buffer issue detected */+LZ4_FORCE_INLINE int LZ4HC_encodeSequence (+ const BYTE** ip,+ BYTE** op,+ const BYTE** anchor,+ int matchLength,+ const BYTE* const match,+ limitedOutput_directive limit,+ BYTE* oend)+{+ size_t length;+ BYTE* const token = (*op)++;++#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6)+ static const BYTE* start = NULL;+ static U32 totalCost = 0;+ U32 const pos = (start==NULL) ? 0 : (U32)(*anchor - start);+ U32 const ll = (U32)(*ip - *anchor);+ U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0;+ U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;+ U32 const cost = 1 + llAdd + ll + 2 + mlAdd;+ if (start==NULL) start = *anchor; /* only works for single segment */+ /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */+ DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",+ pos,+ (U32)(*ip - *anchor), matchLength, (U32)(*ip-match),+ cost, totalCost);+ totalCost += cost;+#endif++ /* Encode Literal length */+ length = (size_t)(*ip - *anchor);+ if ((limit) && ((*op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */+ if (length >= RUN_MASK) {+ size_t len = length - RUN_MASK;+ *token = (RUN_MASK << ML_BITS);+ for(; len >= 255 ; len -= 255) *(*op)++ = 255;+ *(*op)++ = (BYTE)len;+ } else {+ *token = (BYTE)(length << ML_BITS);+ }++ /* Copy Literals */+ LZ4_wildCopy8(*op, *anchor, (*op) + length);+ *op += length;++ /* Encode Offset */+ assert( (*ip - match) <= LZ4_DISTANCE_MAX ); /* note : consider providing offset as a value, rather than as a pointer difference */+ LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;++ /* Encode MatchLength */+ assert(matchLength >= MINMATCH);+ length = (size_t)matchLength - MINMATCH;+ if ((limit) && (*op + (length / 255) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */+ if (length >= ML_MASK) {+ *token += ML_MASK;+ length -= ML_MASK;+ for(; length >= 510 ; length -= 510) { *(*op)++ = 255; *(*op)++ = 255; }+ if (length >= 255) { length -= 255; *(*op)++ = 255; }+ *(*op)++ = (BYTE)length;+ } else {+ *token += (BYTE)(length);+ }++ /* Prepare next loop */+ *ip += matchLength;+ *anchor = *ip;++ return 0;+}++LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (+ LZ4HC_CCtx_internal* const ctx,+ const char* const source,+ char* const dest,+ int* srcSizePtr,+ int const maxOutputSize,+ unsigned maxNbAttempts,+ const limitedOutput_directive limit,+ const dictCtx_directive dict+ )+{+ const int inputSize = *srcSizePtr;+ const int patternAnalysis = (maxNbAttempts > 128); /* levels 9+ */++ const BYTE* ip = (const BYTE*) source;+ const BYTE* anchor = ip;+ const BYTE* const iend = ip + inputSize;+ const BYTE* const mflimit = iend - MFLIMIT;+ const BYTE* const matchlimit = (iend - LASTLITERALS);++ BYTE* optr = (BYTE*) dest;+ BYTE* op = (BYTE*) dest;+ BYTE* oend = op + maxOutputSize;++ int ml0, ml, ml2, ml3;+ const BYTE* start0;+ const BYTE* ref0;+ const BYTE* ref = NULL;+ const BYTE* start2 = NULL;+ const BYTE* ref2 = NULL;+ const BYTE* start3 = NULL;+ const BYTE* ref3 = NULL;++ /* init */+ *srcSizePtr = 0;+ if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */+ if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */++ /* Main Loop */+ while (ip <= mflimit) {+ ml = LZ4HC_InsertAndFindBestMatch(ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict);+ if (ml<MINMATCH) { ip++; continue; }++ /* saved, in case we would skip too much */+ start0 = ip; ref0 = ref; ml0 = ml;++_Search2:+ if (ip+ml <= mflimit) {+ ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,+ ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,+ maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio);+ } else {+ ml2 = ml;+ }++ if (ml2 == ml) { /* No better match => encode ML1 */+ optr = op;+ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow;+ continue;+ }++ if (start0 < ip) { /* first match was skipped at least once */+ if (start2 < ip + ml0) { /* squeezing ML1 between ML0(original ML1) and ML2 */+ ip = start0; ref = ref0; ml = ml0; /* restore initial ML1 */+ } }++ /* Here, start0==ip */+ if ((start2 - ip) < 3) { /* First Match too small : removed */+ ml = ml2;+ ip = start2;+ ref =ref2;+ goto _Search2;+ }++_Search3:+ /* At this stage, we have :+ * ml2 > ml1, and+ * ip1+3 <= ip2 (usually < ip1+ml1) */+ if ((start2 - ip) < OPTIMAL_ML) {+ int correction;+ int new_ml = ml;+ if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;+ if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH;+ correction = new_ml - (int)(start2 - ip);+ if (correction > 0) {+ start2 += correction;+ ref2 += correction;+ ml2 -= correction;+ }+ }+ /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */++ if (start2 + ml2 <= mflimit) {+ ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,+ start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,+ maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio);+ } else {+ ml3 = ml2;+ }++ if (ml3 == ml2) { /* No better match => encode ML1 and ML2 */+ /* ip & ref are known; Now for ml */+ if (start2 < ip+ml) ml = (int)(start2 - ip);+ /* Now, encode 2 sequences */+ optr = op;+ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow;+ ip = start2;+ optr = op;+ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) goto _dest_overflow;+ continue;+ }++ if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */+ if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */+ if (start2 < ip+ml) {+ int correction = (int)(ip+ml - start2);+ start2 += correction;+ ref2 += correction;+ ml2 -= correction;+ if (ml2 < MINMATCH) {+ start2 = start3;+ ref2 = ref3;+ ml2 = ml3;+ }+ }++ optr = op;+ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow;+ ip = start3;+ ref = ref3;+ ml = ml3;++ start0 = start2;+ ref0 = ref2;+ ml0 = ml2;+ goto _Search2;+ }++ start2 = start3;+ ref2 = ref3;+ ml2 = ml3;+ goto _Search3;+ }++ /*+ * OK, now we have 3 ascending matches;+ * let's write the first one ML1.+ * ip & ref are known; Now decide ml.+ */+ if (start2 < ip+ml) {+ if ((start2 - ip) < OPTIMAL_ML) {+ int correction;+ if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;+ if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;+ correction = ml - (int)(start2 - ip);+ if (correction > 0) {+ start2 += correction;+ ref2 += correction;+ ml2 -= correction;+ }+ } else {+ ml = (int)(start2 - ip);+ }+ }+ optr = op;+ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow;++ /* ML2 becomes ML1 */+ ip = start2; ref = ref2; ml = ml2;++ /* ML3 becomes ML2 */+ start2 = start3; ref2 = ref3; ml2 = ml3;++ /* let's find a new ML3 */+ goto _Search3;+ }++_last_literals:+ /* Encode Last Literals */+ { size_t lastRunSize = (size_t)(iend - anchor); /* literals */+ size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;+ size_t const totalSize = 1 + litLength + lastRunSize;+ if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */+ if (limit && (op + totalSize > oend)) {+ if (limit == limitedOutput) return 0; /* Check output limit */+ /* adapt lastRunSize to fill 'dest' */+ lastRunSize = (size_t)(oend - op) - 1;+ litLength = (lastRunSize + 255 - RUN_MASK) / 255;+ lastRunSize -= litLength;+ }+ ip = anchor + lastRunSize;++ if (lastRunSize >= RUN_MASK) {+ size_t accumulator = lastRunSize - RUN_MASK;+ *op++ = (RUN_MASK << ML_BITS);+ for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;+ *op++ = (BYTE) accumulator;+ } else {+ *op++ = (BYTE)(lastRunSize << ML_BITS);+ }+ memcpy(op, anchor, lastRunSize);+ op += lastRunSize;+ }++ /* End */+ *srcSizePtr = (int) (((const char*)ip) - source);+ return (int) (((char*)op)-dest);++_dest_overflow:+ if (limit == fillOutput) {+ op = optr; /* restore correct out pointer */+ goto _last_literals;+ }+ return 0;+}+++static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx,+ const char* const source, char* dst,+ int* srcSizePtr, int dstCapacity,+ int const nbSearches, size_t sufficient_len,+ const limitedOutput_directive limit, int const fullUpdate,+ const dictCtx_directive dict,+ HCfavor_e favorDecSpeed);+++LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (+ LZ4HC_CCtx_internal* const ctx,+ const char* const src,+ char* const dst,+ int* const srcSizePtr,+ int const dstCapacity,+ int cLevel,+ const limitedOutput_directive limit,+ const dictCtx_directive dict+ )+{+ typedef enum { lz4hc, lz4opt } lz4hc_strat_e;+ typedef struct {+ lz4hc_strat_e strat;+ U32 nbSearches;+ U32 targetLength;+ } cParams_t;+ static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = {+ { lz4hc, 2, 16 }, /* 0, unused */+ { lz4hc, 2, 16 }, /* 1, unused */+ { lz4hc, 2, 16 }, /* 2, unused */+ { lz4hc, 4, 16 }, /* 3 */+ { lz4hc, 8, 16 }, /* 4 */+ { lz4hc, 16, 16 }, /* 5 */+ { lz4hc, 32, 16 }, /* 6 */+ { lz4hc, 64, 16 }, /* 7 */+ { lz4hc, 128, 16 }, /* 8 */+ { lz4hc, 256, 16 }, /* 9 */+ { lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/+ { lz4opt, 512,128 }, /*11 */+ { lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */+ };++ DEBUGLOG(4, "LZ4HC_compress_generic(ctx=%p, src=%p, srcSize=%d)", ctx, src, *srcSizePtr);++ if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */+ if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */++ ctx->end += *srcSizePtr;+ if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe something to review */+ cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel);+ { cParams_t const cParam = clTable[cLevel];+ HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio;+ int result;++ if (cParam.strat == lz4hc) {+ result = LZ4HC_compress_hashChain(ctx,+ src, dst, srcSizePtr, dstCapacity,+ cParam.nbSearches, limit, dict);+ } else {+ assert(cParam.strat == lz4opt);+ result = LZ4HC_compress_optimal(ctx,+ src, dst, srcSizePtr, dstCapacity,+ (int)cParam.nbSearches, cParam.targetLength, limit,+ cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */+ dict, favor);+ }+ if (result <= 0) ctx->dirty = 1;+ return result;+ }+}++static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock);++static int+LZ4HC_compress_generic_noDictCtx (+ LZ4HC_CCtx_internal* const ctx,+ const char* const src,+ char* const dst,+ int* const srcSizePtr,+ int const dstCapacity,+ int cLevel,+ limitedOutput_directive limit+ )+{+ assert(ctx->dictCtx == NULL);+ return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx);+}++static int+LZ4HC_compress_generic_dictCtx (+ LZ4HC_CCtx_internal* const ctx,+ const char* const src,+ char* const dst,+ int* const srcSizePtr,+ int const dstCapacity,+ int cLevel,+ limitedOutput_directive limit+ )+{+ const size_t position = (size_t)(ctx->end - ctx->base) - ctx->lowLimit;+ assert(ctx->dictCtx != NULL);+ if (position >= 64 KB) {+ ctx->dictCtx = NULL;+ return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);+ } else if (position == 0 && *srcSizePtr > 4 KB) {+ memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));+ LZ4HC_setExternalDict(ctx, (const BYTE *)src);+ ctx->compressionLevel = (short)cLevel;+ return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);+ } else {+ return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtxHc);+ }+}++static int+LZ4HC_compress_generic (+ LZ4HC_CCtx_internal* const ctx,+ const char* const src,+ char* const dst,+ int* const srcSizePtr,+ int const dstCapacity,+ int cLevel,+ limitedOutput_directive limit+ )+{+ if (ctx->dictCtx == NULL) {+ return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);+ } else {+ return LZ4HC_compress_generic_dictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);+ }+}+++int LZ4_sizeofStateHC(void) { return (int)sizeof(LZ4_streamHC_t); }++#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :+ * it reports an aligment of 8-bytes,+ * while actually aligning LZ4_streamHC_t on 4 bytes. */+static size_t LZ4_streamHC_t_alignment(void)+{+ struct { char c; LZ4_streamHC_t t; } t_a;+ return sizeof(t_a) - sizeof(t_a.t);+}+#endif++/* state is presumed correctly initialized,+ * in which case its size and alignment have already been validate */+int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)+{+ LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;+#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :+ * it reports an aligment of 8-bytes,+ * while actually aligning LZ4_streamHC_t on 4 bytes. */+ assert(((size_t)state & (LZ4_streamHC_t_alignment() - 1)) == 0); /* check alignment */+#endif+ if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */+ LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel);+ LZ4HC_init_internal (ctx, (const BYTE*)src);+ if (dstCapacity < LZ4_compressBound(srcSize))+ return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);+ else+ return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, notLimited);+}++int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)+{+ LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx));+ if (ctx==NULL) return 0; /* init failure */+ return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel);+}++int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)+{+#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1+ LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));+#else+ LZ4_streamHC_t state;+ LZ4_streamHC_t* const statePtr = &state;+#endif+ int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);+#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1+ FREEMEM(statePtr);+#endif+ return cSize;+}++/* state is presumed sized correctly (>= sizeof(LZ4_streamHC_t)) */+int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)+{+ LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx));+ if (ctx==NULL) return 0; /* init failure */+ LZ4HC_init_internal(&ctx->internal_donotuse, (const BYTE*) source);+ LZ4_setCompressionLevel(ctx, cLevel);+ return LZ4HC_compress_generic(&ctx->internal_donotuse, source, dest, sourceSizePtr, targetDestSize, cLevel, fillOutput);+}++++/**************************************+* Streaming Functions+**************************************/+/* allocation */+LZ4_streamHC_t* LZ4_createStreamHC(void)+{+ LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));+ if (LZ4_streamHCPtr==NULL) return NULL;+ LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); /* full initialization, malloc'ed buffer can be full of garbage */+ return LZ4_streamHCPtr;+}++int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr)+{+ DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr);+ if (!LZ4_streamHCPtr) return 0; /* support free on NULL */+ FREEMEM(LZ4_streamHCPtr);+ return 0;+}+++LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size)+{+ LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer;+ if (buffer == NULL) return NULL;+ if (size < sizeof(LZ4_streamHC_t)) return NULL;+#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :+ * it reports an aligment of 8-bytes,+ * while actually aligning LZ4_streamHC_t on 4 bytes. */+ if (((size_t)buffer) & (LZ4_streamHC_t_alignment() - 1)) return NULL; /* alignment check */+#endif+ /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */+ LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= LZ4_STREAMHCSIZE);+ DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", LZ4_streamHCPtr, (unsigned)size);+ /* end-base will trigger a clearTable on starting compression */+ LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;+ LZ4_streamHCPtr->internal_donotuse.base = NULL;+ LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;+ LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = 0;+ LZ4_streamHCPtr->internal_donotuse.dirty = 0;+ LZ4_setCompressionLevel(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT);+ return LZ4_streamHCPtr;+}++/* just a stub */+void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)+{+ LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr));+ LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);+}++void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)+{+ DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel);+ if (LZ4_streamHCPtr->internal_donotuse.dirty) {+ LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr));+ } else {+ /* preserve end - base : can trigger clearTable's threshold */+ LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;+ LZ4_streamHCPtr->internal_donotuse.base = NULL;+ LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;+ }+ LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);+}++void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)+{+ DEBUGLOG(5, "LZ4_setCompressionLevel(%p, %d)", LZ4_streamHCPtr, compressionLevel);+ if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;+ if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX;+ LZ4_streamHCPtr->internal_donotuse.compressionLevel = (short)compressionLevel;+}++void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor)+{+ LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor!=0);+}++/* LZ4_loadDictHC() :+ * LZ4_streamHCPtr is presumed properly initialized */+int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr,+ const char* dictionary, int dictSize)+{+ LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;+ DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize);+ assert(LZ4_streamHCPtr != NULL);+ if (dictSize > 64 KB) {+ dictionary += (size_t)dictSize - 64 KB;+ dictSize = 64 KB;+ }+ /* need a full initialization, there are bad side-effects when using resetFast() */+ { int const cLevel = ctxPtr->compressionLevel;+ LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr));+ LZ4_setCompressionLevel(LZ4_streamHCPtr, cLevel);+ }+ LZ4HC_init_internal (ctxPtr, (const BYTE*)dictionary);+ ctxPtr->end = (const BYTE*)dictionary + dictSize;+ if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);+ return dictSize;+}++void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream) {+ working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL;+}++/* compression */++static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)+{+ DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);+ if (ctxPtr->end >= ctxPtr->base + ctxPtr->dictLimit + 4)+ LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */++ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */+ ctxPtr->lowLimit = ctxPtr->dictLimit;+ ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);+ ctxPtr->dictBase = ctxPtr->base;+ ctxPtr->base = newBlock - ctxPtr->dictLimit;+ ctxPtr->end = newBlock;+ ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */++ /* cannot reference an extDict and a dictCtx at the same time */+ ctxPtr->dictCtx = NULL;+}++static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,+ const char* src, char* dst,+ int* srcSizePtr, int dstCapacity,+ limitedOutput_directive limit)+{+ LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;+ DEBUGLOG(4, "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d)",+ LZ4_streamHCPtr, src, *srcSizePtr);+ assert(ctxPtr != NULL);+ /* auto-init if forgotten */+ if (ctxPtr->base == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src);++ /* Check overflow */+ if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) {+ size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;+ if (dictSize > 64 KB) dictSize = 64 KB;+ LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize);+ }++ /* Check if blocks follow each other */+ if ((const BYTE*)src != ctxPtr->end)+ LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src);++ /* Check overlapping input/dictionary space */+ { const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr;+ const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;+ const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;+ if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) {+ if (sourceEnd > dictEnd) sourceEnd = dictEnd;+ ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);+ if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;+ }+ }++ return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit);+}++int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity)+{+ if (dstCapacity < LZ4_compressBound(srcSize))+ return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput);+ else+ return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, notLimited);+}++int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize)+{+ return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput);+}++++/* dictionary saving */++int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)+{+ LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;+ int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));+ DEBUGLOG(4, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);+ if (dictSize > 64 KB) dictSize = 64 KB;+ if (dictSize < 4) dictSize = 0;+ if (dictSize > prefixSize) dictSize = prefixSize;+ memmove(safeBuffer, streamPtr->end - dictSize, dictSize);+ { U32 const endIndex = (U32)(streamPtr->end - streamPtr->base);+ streamPtr->end = (const BYTE*)safeBuffer + dictSize;+ streamPtr->base = streamPtr->end - endIndex;+ streamPtr->dictLimit = endIndex - (U32)dictSize;+ streamPtr->lowLimit = endIndex - (U32)dictSize;+ if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit;+ }+ return dictSize;+}+++/***************************************************+* Deprecated Functions+***************************************************/++/* These functions currently generate deprecation warnings */++/* Wrappers for deprecated compression functions */+int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); }+int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); }+int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); }+int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); }+int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); }+int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); }+int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); }+int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); }+int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); }+int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); }+++/* Deprecated streaming functions */+int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; }++/* state is presumed correctly sized, aka >= sizeof(LZ4_streamHC_t)+ * @return : 0 on success, !=0 if error */+int LZ4_resetStreamStateHC(void* state, char* inputBuffer)+{+ LZ4_streamHC_t* const hc4 = LZ4_initStreamHC(state, sizeof(*hc4));+ if (hc4 == NULL) return 1; /* init failed */+ LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer);+ return 0;+}++void* LZ4_createHC (const char* inputBuffer)+{+ LZ4_streamHC_t* const hc4 = LZ4_createStreamHC();+ if (hc4 == NULL) return NULL; /* not enough memory */+ LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer);+ return hc4;+}++int LZ4_freeHC (void* LZ4HC_Data)+{+ if (!LZ4HC_Data) return 0; /* support free on NULL */+ FREEMEM(LZ4HC_Data);+ return 0;+}++int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel)+{+ return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, notLimited);+}++int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel)+{+ return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, dstCapacity, cLevel, limitedOutput);+}++char* LZ4_slideInputBufferHC(void* LZ4HC_Data)+{+ LZ4_streamHC_t *ctx = (LZ4_streamHC_t*)LZ4HC_Data;+ const BYTE *bufferStart = ctx->internal_donotuse.base + ctx->internal_donotuse.lowLimit;+ LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel);+ /* avoid const char * -> char * conversion warning :( */+ return (char *)(uptrval)bufferStart;+}+++/* ================================================+ * LZ4 Optimal parser (levels [LZ4HC_CLEVEL_OPT_MIN - LZ4HC_CLEVEL_MAX])+ * ===============================================*/+typedef struct {+ int price;+ int off;+ int mlen;+ int litlen;+} LZ4HC_optimal_t;++/* price in bytes */+LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen)+{+ int price = litlen;+ assert(litlen >= 0);+ if (litlen >= (int)RUN_MASK)+ price += 1 + ((litlen-(int)RUN_MASK) / 255);+ return price;+}+++/* requires mlen >= MINMATCH */+LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)+{+ int price = 1 + 2 ; /* token + 16-bit offset */+ assert(litlen >= 0);+ assert(mlen >= MINMATCH);++ price += LZ4HC_literalsPrice(litlen);++ if (mlen >= (int)(ML_MASK+MINMATCH))+ price += 1 + ((mlen-(int)(ML_MASK+MINMATCH)) / 255);++ return price;+}+++typedef struct {+ int off;+ int len;+} LZ4HC_match_t;++LZ4_FORCE_INLINE LZ4HC_match_t+LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,+ const BYTE* ip, const BYTE* const iHighLimit,+ int minLen, int nbSearches,+ const dictCtx_directive dict,+ const HCfavor_e favorDecSpeed)+{+ LZ4HC_match_t match = { 0 , 0 };+ const BYTE* matchPtr = NULL;+ /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),+ * but this won't be the case here, as we define iLowLimit==ip,+ * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */+ int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed);+ if (matchLength <= minLen) return match;+ if (favorDecSpeed) {+ if ((matchLength>18) & (matchLength<=36)) matchLength=18; /* favor shortcut */+ }+ match.len = matchLength;+ match.off = (int)(ip-matchPtr);+ return match;+}+++static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,+ const char* const source,+ char* dst,+ int* srcSizePtr,+ int dstCapacity,+ int const nbSearches,+ size_t sufficient_len,+ const limitedOutput_directive limit,+ int const fullUpdate,+ const dictCtx_directive dict,+ const HCfavor_e favorDecSpeed)+{+#define TRAILING_LITERALS 3+ LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */++ const BYTE* ip = (const BYTE*) source;+ const BYTE* anchor = ip;+ const BYTE* const iend = ip + *srcSizePtr;+ const BYTE* const mflimit = iend - MFLIMIT;+ const BYTE* const matchlimit = iend - LASTLITERALS;+ BYTE* op = (BYTE*) dst;+ BYTE* opSaved = (BYTE*) dst;+ BYTE* oend = op + dstCapacity;++ /* init */+ DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity);+ *srcSizePtr = 0;+ if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */+ if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;++ /* Main Loop */+ assert(ip - anchor < LZ4_MAX_INPUT_SIZE);+ while (ip <= mflimit) {+ int const llen = (int)(ip - anchor);+ int best_mlen, best_off;+ int cur, last_match_pos = 0;++ LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed);+ if (firstMatch.len==0) { ip++; continue; }++ if ((size_t)firstMatch.len > sufficient_len) {+ /* good enough solution : immediate encoding */+ int const firstML = firstMatch.len;+ const BYTE* const matchPos = ip - firstMatch.off;+ opSaved = op;+ if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */+ goto _dest_overflow;+ continue;+ }++ /* set prices for first positions (literals) */+ { int rPos;+ for (rPos = 0 ; rPos < MINMATCH ; rPos++) {+ int const cost = LZ4HC_literalsPrice(llen + rPos);+ opt[rPos].mlen = 1;+ opt[rPos].off = 0;+ opt[rPos].litlen = llen + rPos;+ opt[rPos].price = cost;+ DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",+ rPos, cost, opt[rPos].litlen);+ } }+ /* set prices using initial match */+ { int mlen = MINMATCH;+ int const matchML = firstMatch.len; /* necessarily < sufficient_len < LZ4_OPT_NUM */+ int const offset = firstMatch.off;+ assert(matchML < LZ4_OPT_NUM);+ for ( ; mlen <= matchML ; mlen++) {+ int const cost = LZ4HC_sequencePrice(llen, mlen);+ opt[mlen].mlen = mlen;+ opt[mlen].off = offset;+ opt[mlen].litlen = llen;+ opt[mlen].price = cost;+ DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup",+ mlen, cost, mlen);+ } }+ last_match_pos = firstMatch.len;+ { int addLit;+ for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {+ opt[last_match_pos+addLit].mlen = 1; /* literal */+ opt[last_match_pos+addLit].off = 0;+ opt[last_match_pos+addLit].litlen = addLit;+ opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);+ DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",+ last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);+ } }++ /* check further positions */+ for (cur = 1; cur < last_match_pos; cur++) {+ const BYTE* const curPtr = ip + cur;+ LZ4HC_match_t newMatch;++ if (curPtr > mflimit) break;+ DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",+ cur, opt[cur].price, opt[cur+1].price, cur+1);+ if (fullUpdate) {+ /* not useful to search here if next position has same (or lower) cost */+ if ( (opt[cur+1].price <= opt[cur].price)+ /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */+ && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) )+ continue;+ } else {+ /* not useful to search here if next position has same (or lower) cost */+ if (opt[cur+1].price <= opt[cur].price) continue;+ }++ DEBUGLOG(7, "search at rPos:%u", cur);+ if (fullUpdate)+ newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed);+ else+ /* only test matches of minimum length; slightly faster, but misses a few bytes */+ newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict, favorDecSpeed);+ if (!newMatch.len) continue;++ if ( ((size_t)newMatch.len > sufficient_len)+ || (newMatch.len + cur >= LZ4_OPT_NUM) ) {+ /* immediate encoding */+ best_mlen = newMatch.len;+ best_off = newMatch.off;+ last_match_pos = cur + 1;+ goto encode;+ }++ /* before match : set price with literals at beginning */+ { int const baseLitlen = opt[cur].litlen;+ int litlen;+ for (litlen = 1; litlen < MINMATCH; litlen++) {+ int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen);+ int const pos = cur + litlen;+ if (price < opt[pos].price) {+ opt[pos].mlen = 1; /* literal */+ opt[pos].off = 0;+ opt[pos].litlen = baseLitlen+litlen;+ opt[pos].price = price;+ DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)",+ pos, price, opt[pos].litlen);+ } } }++ /* set prices using match at position = cur */+ { int const matchML = newMatch.len;+ int ml = MINMATCH;++ assert(cur + newMatch.len < LZ4_OPT_NUM);+ for ( ; ml <= matchML ; ml++) {+ int const pos = cur + ml;+ int const offset = newMatch.off;+ int price;+ int ll;+ DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)",+ pos, last_match_pos);+ if (opt[cur].mlen == 1) {+ ll = opt[cur].litlen;+ price = ((cur > ll) ? opt[cur - ll].price : 0)+ + LZ4HC_sequencePrice(ll, ml);+ } else {+ ll = 0;+ price = opt[cur].price + LZ4HC_sequencePrice(0, ml);+ }++ assert((U32)favorDecSpeed <= 1);+ if (pos > last_match_pos+TRAILING_LITERALS+ || price <= opt[pos].price - (int)favorDecSpeed) {+ DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",+ pos, price, ml);+ assert(pos < LZ4_OPT_NUM);+ if ( (ml == matchML) /* last pos of last match */+ && (last_match_pos < pos) )+ last_match_pos = pos;+ opt[pos].mlen = ml;+ opt[pos].off = offset;+ opt[pos].litlen = ll;+ opt[pos].price = price;+ } } }+ /* complete following positions with literals */+ { int addLit;+ for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {+ opt[last_match_pos+addLit].mlen = 1; /* literal */+ opt[last_match_pos+addLit].off = 0;+ opt[last_match_pos+addLit].litlen = addLit;+ opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);+ DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);+ } }+ } /* for (cur = 1; cur <= last_match_pos; cur++) */++ assert(last_match_pos < LZ4_OPT_NUM + TRAILING_LITERALS);+ best_mlen = opt[last_match_pos].mlen;+ best_off = opt[last_match_pos].off;+ cur = last_match_pos - best_mlen;++ encode: /* cur, last_match_pos, best_mlen, best_off must be set */+ assert(cur < LZ4_OPT_NUM);+ assert(last_match_pos >= 1); /* == 1 when only one candidate */+ DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos);+ { int candidate_pos = cur;+ int selected_matchLength = best_mlen;+ int selected_offset = best_off;+ while (1) { /* from end to beginning */+ int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */+ int const next_offset = opt[candidate_pos].off;+ DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength);+ opt[candidate_pos].mlen = selected_matchLength;+ opt[candidate_pos].off = selected_offset;+ selected_matchLength = next_matchLength;+ selected_offset = next_offset;+ if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */+ assert(next_matchLength > 0); /* can be 1, means literal */+ candidate_pos -= next_matchLength;+ } }++ /* encode all recorded sequences in order */+ { int rPos = 0; /* relative position (to ip) */+ while (rPos < last_match_pos) {+ int const ml = opt[rPos].mlen;+ int const offset = opt[rPos].off;+ if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */+ rPos += ml;+ assert(ml >= MINMATCH);+ assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX));+ opSaved = op;+ if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */+ goto _dest_overflow;+ } }+ } /* while (ip <= mflimit) */++ _last_literals:+ /* Encode Last Literals */+ { size_t lastRunSize = (size_t)(iend - anchor); /* literals */+ size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;+ size_t const totalSize = 1 + litLength + lastRunSize;+ if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */+ if (limit && (op + totalSize > oend)) {+ if (limit == limitedOutput) return 0; /* Check output limit */+ /* adapt lastRunSize to fill 'dst' */+ lastRunSize = (size_t)(oend - op) - 1;+ litLength = (lastRunSize + 255 - RUN_MASK) / 255;+ lastRunSize -= litLength;+ }+ ip = anchor + lastRunSize;++ if (lastRunSize >= RUN_MASK) {+ size_t accumulator = lastRunSize - RUN_MASK;+ *op++ = (RUN_MASK << ML_BITS);+ for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;+ *op++ = (BYTE) accumulator;+ } else {+ *op++ = (BYTE)(lastRunSize << ML_BITS);+ }+ memcpy(op, anchor, lastRunSize);+ op += lastRunSize;+ }++ /* End */+ *srcSizePtr = (int) (((const char*)ip) - source);+ return (int) ((char*)op-dst);++ _dest_overflow:+ if (limit == fillOutput) {+ op = opSaved; /* restore correct out pointer */+ goto _last_literals;+ }+ return 0;+ }
+ lz4/lib/lz4hc.h view
@@ -0,0 +1,438 @@+/*+ LZ4 HC - High Compression Mode of LZ4+ Header File+ Copyright (C) 2011-2017, Yann Collet.+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)++ Redistribution and use in source and binary forms, with or without+ modification, are permitted provided that the following conditions are+ met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.+ * Redistributions in binary form must reproduce the above+ copyright notice, this list of conditions and the following disclaimer+ in the documentation and/or other materials provided with the+ distribution.++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.++ You can contact the author at :+ - LZ4 source repository : https://github.com/lz4/lz4+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c+*/+#ifndef LZ4_HC_H_19834876238432+#define LZ4_HC_H_19834876238432++#if defined (__cplusplus)+extern "C" {+#endif++/* --- Dependency --- */+/* note : lz4hc requires lz4.h/lz4.c for compilation */+#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */+++/* --- Useful constants --- */+#define LZ4HC_CLEVEL_MIN 3+#define LZ4HC_CLEVEL_DEFAULT 9+#define LZ4HC_CLEVEL_OPT_MIN 10+#define LZ4HC_CLEVEL_MAX 12+++/*-************************************+ * Block Compression+ **************************************/+/*! LZ4_compress_HC() :+ * Compress data from `src` into `dst`, using the powerful but slower "HC" algorithm.+ * `dst` must be already allocated.+ * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h")+ * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h")+ * `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work.+ * Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX.+ * @return : the number of bytes written into 'dst'+ * or 0 if compression fails.+ */+LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);+++/* Note :+ * Decompression functions are provided within "lz4.h" (BSD license)+ */+++/*! LZ4_compress_HC_extStateHC() :+ * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`.+ * `state` size is provided by LZ4_sizeofStateHC().+ * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly).+ */+LZ4LIB_API int LZ4_sizeofStateHC(void);+LZ4LIB_API int LZ4_compress_HC_extStateHC(void* stateHC, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);+++/*! LZ4_compress_HC_destSize() : v1.9.0++ * Will compress as much data as possible from `src`+ * to fit into `targetDstSize` budget.+ * Result is provided in 2 parts :+ * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize)+ * or 0 if compression fails.+ * `srcSizePtr` : on success, *srcSizePtr is updated to indicate how much bytes were read from `src`+ */+LZ4LIB_API int LZ4_compress_HC_destSize(void* stateHC,+ const char* src, char* dst,+ int* srcSizePtr, int targetDstSize,+ int compressionLevel);+++/*-************************************+ * Streaming Compression+ * Bufferless synchronous API+ **************************************/+ typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */++/*! LZ4_createStreamHC() and LZ4_freeStreamHC() :+ * These functions create and release memory for LZ4 HC streaming state.+ * Newly created states are automatically initialized.+ * A same state can be used multiple times consecutively,+ * starting with LZ4_resetStreamHC_fast() to start a new stream of blocks.+ */+LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void);+LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);++/*+ These functions compress data in successive blocks of any size,+ using previous blocks as dictionary, to improve compression ratio.+ One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks.+ There is an exception for ring buffers, which can be smaller than 64 KB.+ Ring-buffer scenario is automatically detected and handled within LZ4_compress_HC_continue().++ Before starting compression, state must be allocated and properly initialized.+ LZ4_createStreamHC() does both, though compression level is set to LZ4HC_CLEVEL_DEFAULT.++ Selecting the compression level can be done with LZ4_resetStreamHC_fast() (starts a new stream)+ or LZ4_setCompressionLevel() (anytime, between blocks in the same stream) (experimental).+ LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once,+ which is automatically the case when state is created using LZ4_createStreamHC().++ After reset, a first "fictional block" can be designated as initial dictionary,+ using LZ4_loadDictHC() (Optional).++ Invoke LZ4_compress_HC_continue() to compress each successive block.+ The number of blocks is unlimited.+ Previous input blocks, including initial dictionary when present,+ must remain accessible and unmodified during compression.++ It's allowed to update compression level anytime between blocks,+ using LZ4_setCompressionLevel() (experimental).++ 'dst' buffer should be sized to handle worst case scenarios+ (see LZ4_compressBound(), it ensures compression success).+ In case of failure, the API does not guarantee recovery,+ so the state _must_ be reset.+ To ensure compression success+ whenever `dst` buffer size cannot be made >= LZ4_compressBound(),+ consider using LZ4_compress_HC_continue_destSize().++ Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks,+ it's possible to copy the last blocks into a more stable memory space, using LZ4_saveDictHC().+ Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer' (<= 64 KB)++ After completing a streaming compression,+ it's possible to start a new stream of blocks, using the same LZ4_streamHC_t state,+ just by resetting it, using LZ4_resetStreamHC_fast().+*/++LZ4LIB_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t* streamHCPtr, int compressionLevel); /* v1.9.0+ */+LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);++LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr,+ const char* src, char* dst,+ int srcSize, int maxDstSize);++/*! LZ4_compress_HC_continue_destSize() : v1.9.0++ * Similar to LZ4_compress_HC_continue(),+ * but will read as much data as possible from `src`+ * to fit into `targetDstSize` budget.+ * Result is provided into 2 parts :+ * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize)+ * or 0 if compression fails.+ * `srcSizePtr` : on success, *srcSizePtr will be updated to indicate how much bytes were read from `src`.+ * Note that this function may not consume the entire input.+ */+LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,+ const char* src, char* dst,+ int* srcSizePtr, int targetDstSize);++LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);++++/*^**********************************************+ * !!!!!! STATIC LINKING ONLY !!!!!!+ ***********************************************/++/*-******************************************************************+ * PRIVATE DEFINITIONS :+ * Do not use these definitions directly.+ * They are merely exposed to allow static allocation of `LZ4_streamHC_t`.+ * Declare an `LZ4_streamHC_t` directly, rather than any type below.+ * Even then, only do so in the context of static linking, as definitions may change between versions.+ ********************************************************************/++#define LZ4HC_DICTIONARY_LOGSIZE 16+#define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE)+#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1)++#define LZ4HC_HASH_LOG 15+#define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG)+#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1)+++#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)+#include <stdint.h>++typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;+struct LZ4HC_CCtx_internal+{+ uint32_t hashTable[LZ4HC_HASHTABLESIZE];+ uint16_t chainTable[LZ4HC_MAXD];+ const uint8_t* end; /* next block here to continue on current prefix */+ const uint8_t* base; /* All index relative to this position */+ const uint8_t* dictBase; /* alternate base for extDict */+ uint32_t dictLimit; /* below that point, need extDict */+ uint32_t lowLimit; /* below that point, no more dict */+ uint32_t nextToUpdate; /* index from which to continue dictionary update */+ short compressionLevel;+ int8_t favorDecSpeed; /* favor decompression speed if this flag set,+ otherwise, favor compression ratio */+ int8_t dirty; /* stream has to be fully reset if this flag is set */+ const LZ4HC_CCtx_internal* dictCtx;+};++#else++typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;+struct LZ4HC_CCtx_internal+{+ unsigned int hashTable[LZ4HC_HASHTABLESIZE];+ unsigned short chainTable[LZ4HC_MAXD];+ const unsigned char* end; /* next block here to continue on current prefix */+ const unsigned char* base; /* All index relative to this position */+ const unsigned char* dictBase; /* alternate base for extDict */+ unsigned int dictLimit; /* below that point, need extDict */+ unsigned int lowLimit; /* below that point, no more dict */+ unsigned int nextToUpdate; /* index from which to continue dictionary update */+ short compressionLevel;+ char favorDecSpeed; /* favor decompression speed if this flag set,+ otherwise, favor compression ratio */+ char dirty; /* stream has to be fully reset if this flag is set */+ const LZ4HC_CCtx_internal* dictCtx;+};++#endif+++/* Do not use these definitions directly !+ * Declare or allocate an LZ4_streamHC_t instead.+ */+#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56 + ((sizeof(void*)==16) ? 56 : 0) /* AS400*/ ) /* 262200 or 262256*/+#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))+union LZ4_streamHC_u {+ size_t table[LZ4_STREAMHCSIZE_SIZET];+ LZ4HC_CCtx_internal internal_donotuse;+}; /* previously typedef'd to LZ4_streamHC_t */++/* LZ4_streamHC_t :+ * This structure allows static allocation of LZ4 HC streaming state.+ * This can be used to allocate statically, on state, or as part of a larger structure.+ *+ * Such state **must** be initialized using LZ4_initStreamHC() before first use.+ *+ * Note that invoking LZ4_initStreamHC() is not required when+ * the state was created using LZ4_createStreamHC() (which is recommended).+ * Using the normal builder, a newly created state is automatically initialized.+ *+ * Static allocation shall only be used in combination with static linking.+ */++/* LZ4_initStreamHC() : v1.9.0++ * Required before first use of a statically allocated LZ4_streamHC_t.+ * Before v1.9.0 : use LZ4_resetStreamHC() instead+ */+LZ4LIB_API LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size);+++/*-************************************+* Deprecated Functions+**************************************/+/* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */++/* deprecated compression functions */+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC (const char* source, char* dest, int inputSize);+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);++/* Obsolete streaming functions; degraded functionality; do not use!+ *+ * In order to perform streaming compression, these functions depended on data+ * that is no longer tracked in the state. They have been preserved as well as+ * possible: using them will still produce a correct output. However, use of+ * LZ4_slideInputBufferHC() will truncate the history of the stream, rather+ * than preserve a window-sized chunk of history.+ */+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);+LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data);+LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data);+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API int LZ4_sizeofStreamStateHC(void);+LZ4_DEPRECATED("use LZ4_initStreamHC() instead") LZ4LIB_API int LZ4_resetStreamStateHC(void* state, char* inputBuffer);+++/* LZ4_resetStreamHC() is now replaced by LZ4_initStreamHC().+ * The intention is to emphasize the difference with LZ4_resetStreamHC_fast(),+ * which is now the recommended function to start a new stream of blocks,+ * but cannot be used to initialize a memory segment containing arbitrary garbage data.+ *+ * It is recommended to switch to LZ4_initStreamHC().+ * LZ4_resetStreamHC() will generate deprecation warnings in a future version.+ */+LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel);+++#if defined (__cplusplus)+}+#endif++#endif /* LZ4_HC_H_19834876238432 */+++/*-**************************************************+ * !!!!! STATIC LINKING ONLY !!!!!+ * Following definitions are considered experimental.+ * They should not be linked from DLL,+ * as there is no guarantee of API stability yet.+ * Prototypes will be promoted to "stable" status+ * after successfull usage in real-life scenarios.+ ***************************************************/+#ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */+#ifndef LZ4_HC_SLO_098092834+#define LZ4_HC_SLO_098092834++#define LZ4_STATIC_LINKING_ONLY /* LZ4LIB_STATIC_API */+#include "lz4.h"++#if defined (__cplusplus)+extern "C" {+#endif++/*! LZ4_setCompressionLevel() : v1.8.0+ (experimental)+ * It's possible to change compression level+ * between successive invocations of LZ4_compress_HC_continue*()+ * for dynamic adaptation.+ */+LZ4LIB_STATIC_API void LZ4_setCompressionLevel(+ LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);++/*! LZ4_favorDecompressionSpeed() : v1.8.2+ (experimental)+ * Opt. Parser will favor decompression speed over compression ratio.+ * Only applicable to levels >= LZ4HC_CLEVEL_OPT_MIN.+ */+LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed(+ LZ4_streamHC_t* LZ4_streamHCPtr, int favor);++/*! LZ4_resetStreamHC_fast() : v1.9.0++ * When an LZ4_streamHC_t is known to be in a internally coherent state,+ * it can often be prepared for a new compression with almost no work, only+ * sometimes falling back to the full, expensive reset that is always required+ * when the stream is in an indeterminate state (i.e., the reset performed by+ * LZ4_resetStreamHC()).+ *+ * LZ4_streamHCs are guaranteed to be in a valid state when:+ * - returned from LZ4_createStreamHC()+ * - reset by LZ4_resetStreamHC()+ * - memset(stream, 0, sizeof(LZ4_streamHC_t))+ * - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast()+ * - the stream was in a valid state and was then used in any compression call+ * that returned success+ * - the stream was in an indeterminate state and was used in a compression+ * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that+ * returned success+ *+ * Note:+ * A stream that was last used in a compression call that returned an error+ * may be passed to this function. However, it will be fully reset, which will+ * clear any existing history and settings from the context.+ */+LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast(+ LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);++/*! LZ4_compress_HC_extStateHC_fastReset() :+ * A variant of LZ4_compress_HC_extStateHC().+ *+ * Using this variant avoids an expensive initialization step. It is only safe+ * to call if the state buffer is known to be correctly initialized already+ * (see above comment on LZ4_resetStreamHC_fast() for a definition of+ * "correctly initialized"). From a high level, the difference is that this+ * function initializes the provided state with a call to+ * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a+ * call to LZ4_resetStreamHC().+ */+LZ4LIB_STATIC_API int LZ4_compress_HC_extStateHC_fastReset (+ void* state,+ const char* src, char* dst,+ int srcSize, int dstCapacity,+ int compressionLevel);++/*! LZ4_attach_HC_dictionary() :+ * This is an experimental API that allows for the efficient use of a+ * static dictionary many times.+ *+ * Rather than re-loading the dictionary buffer into a working context before+ * each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a+ * working LZ4_streamHC_t, this function introduces a no-copy setup mechanism,+ * in which the working stream references the dictionary stream in-place.+ *+ * Several assumptions are made about the state of the dictionary stream.+ * Currently, only streams which have been prepared by LZ4_loadDictHC() should+ * be expected to work.+ *+ * Alternatively, the provided dictionary stream pointer may be NULL, in which+ * case any existing dictionary stream is unset.+ *+ * A dictionary should only be attached to a stream without any history (i.e.,+ * a stream that has just been reset).+ *+ * The dictionary will remain attached to the working stream only for the+ * current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the+ * dictionary context association from the working stream. The dictionary+ * stream (and source buffer) must remain in-place / accessible / unchanged+ * through the lifetime of the stream session.+ */+LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary(+ LZ4_streamHC_t *working_stream,+ const LZ4_streamHC_t *dictionary_stream);++#if defined (__cplusplus)+}+#endif++#endif /* LZ4_HC_SLO_098092834 */+#endif /* LZ4_HC_STATIC_LINKING_ONLY */
+ lz4/lib/xxhash.c view
@@ -0,0 +1,1030 @@+/*+* xxHash - Fast Hash algorithm+* Copyright (C) 2012-2016, Yann Collet+*+* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)+*+* Redistribution and use in source and binary forms, with or without+* modification, are permitted provided that the following conditions are+* met:+*+* * Redistributions of source code must retain the above copyright+* notice, this list of conditions and the following disclaimer.+* * Redistributions in binary form must reproduce the above+* copyright notice, this list of conditions and the following disclaimer+* in the documentation and/or other materials provided with the+* distribution.+*+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+*+* You can contact the author at :+* - xxHash homepage: http://www.xxhash.com+* - xxHash source repository : https://github.com/Cyan4973/xxHash+*/+++/* *************************************+* Tuning parameters+***************************************/+/*!XXH_FORCE_MEMORY_ACCESS :+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.+ * The below switch allow to select different access method for improved performance.+ * Method 0 (default) : use `memcpy()`. Safe and portable.+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).+ * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.+ * Method 2 : direct access. This method doesn't depend on compiler but violate C standard.+ * It can generate buggy code on targets which do not support unaligned memory accesses.+ * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)+ * See http://stackoverflow.com/a/32095106/646947 for details.+ * Prefer these methods in priority order (0 > 1 > 2)+ */+#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */+# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \+ || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \+ || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )+# define XXH_FORCE_MEMORY_ACCESS 2+# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \+ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \+ || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \+ || defined(__ARM_ARCH_7S__) ))+# define XXH_FORCE_MEMORY_ACCESS 1+# endif+#endif++/*!XXH_ACCEPT_NULL_INPUT_POINTER :+ * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault.+ * When this macro is enabled, xxHash actively checks input for null pointer.+ * It it is, result for null input pointers is the same as a null-length input.+ */+#ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */+# define XXH_ACCEPT_NULL_INPUT_POINTER 0+#endif++/*!XXH_FORCE_NATIVE_FORMAT :+ * By default, xxHash library provides endian-independent Hash values, based on little-endian convention.+ * Results are therefore identical for little-endian and big-endian CPU.+ * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.+ * Should endian-independence be of no importance for your application, you may set the #define below to 1,+ * to improve speed for Big-endian CPU.+ * This option has no impact on Little_Endian CPU.+ */+#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */+# define XXH_FORCE_NATIVE_FORMAT 0+#endif++/*!XXH_FORCE_ALIGN_CHECK :+ * This is a minor performance trick, only useful with lots of very small keys.+ * It means : check for aligned/unaligned input.+ * The check costs one initial branch per hash;+ * set it to 0 when the input is guaranteed to be aligned,+ * or when alignment doesn't matter for performance.+ */+#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */+# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)+# define XXH_FORCE_ALIGN_CHECK 0+# else+# define XXH_FORCE_ALIGN_CHECK 1+# endif+#endif+++/* *************************************+* Includes & Memory related functions+***************************************/+/*! Modify the local functions below should you wish to use some other memory routines+* for malloc(), free() */+#include <stdlib.h>+static void* XXH_malloc(size_t s) { return malloc(s); }+static void XXH_free (void* p) { free(p); }+/*! and for memcpy() */+#include <string.h>+static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }++#include <assert.h> /* assert */++#define XXH_STATIC_LINKING_ONLY+#include "xxhash.h"+++/* *************************************+* Compiler Specific Options+***************************************/+#ifdef _MSC_VER /* Visual Studio */+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */+# define FORCE_INLINE static __forceinline+#else+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */+# ifdef __GNUC__+# define FORCE_INLINE static inline __attribute__((always_inline))+# else+# define FORCE_INLINE static inline+# endif+# else+# define FORCE_INLINE static+# endif /* __STDC_VERSION__ */+#endif+++/* *************************************+* Basic Types+***************************************/+#ifndef MEM_MODULE+# if !defined (__VMS) \+ && (defined (__cplusplus) \+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )+# include <stdint.h>+ typedef uint8_t BYTE;+ typedef uint16_t U16;+ typedef uint32_t U32;+# else+ typedef unsigned char BYTE;+ typedef unsigned short U16;+ typedef unsigned int U32;+# endif+#endif++#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))++/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */+static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }++#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))++/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */+/* currently only defined for gcc and icc */+typedef union { U32 u32; } __attribute__((packed)) unalign;+static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }++#else++/* portable and safe solution. Generally efficient.+ * see : http://stackoverflow.com/a/32095106/646947+ */+static U32 XXH_read32(const void* memPtr)+{+ U32 val;+ memcpy(&val, memPtr, sizeof(val));+ return val;+}++#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */+++/* ****************************************+* Compiler-specific Functions and Macros+******************************************/+#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)++/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */+#if defined(_MSC_VER)+# define XXH_rotl32(x,r) _rotl(x,r)+# define XXH_rotl64(x,r) _rotl64(x,r)+#else+# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))+# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))+#endif++#if defined(_MSC_VER) /* Visual Studio */+# define XXH_swap32 _byteswap_ulong+#elif XXH_GCC_VERSION >= 403+# define XXH_swap32 __builtin_bswap32+#else+static U32 XXH_swap32 (U32 x)+{+ return ((x << 24) & 0xff000000 ) |+ ((x << 8) & 0x00ff0000 ) |+ ((x >> 8) & 0x0000ff00 ) |+ ((x >> 24) & 0x000000ff );+}+#endif+++/* *************************************+* Architecture Macros+***************************************/+typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;++/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */+#ifndef XXH_CPU_LITTLE_ENDIAN+static int XXH_isLittleEndian(void)+{+ const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */+ return one.c[0];+}+# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian()+#endif+++/* ***************************+* Memory reads+*****************************/+typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;++FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)+{+ if (align==XXH_unaligned)+ return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));+ else+ return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);+}++FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)+{+ return XXH_readLE32_align(ptr, endian, XXH_unaligned);+}++static U32 XXH_readBE32(const void* ptr)+{+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);+}+++/* *************************************+* Macros+***************************************/+#define XXH_STATIC_ASSERT(c) { enum { XXH_sa = 1/(int)(!!(c)) }; } /* use after variable declarations */+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }+++/* *******************************************************************+* 32-bit hash functions+*********************************************************************/+static const U32 PRIME32_1 = 2654435761U;+static const U32 PRIME32_2 = 2246822519U;+static const U32 PRIME32_3 = 3266489917U;+static const U32 PRIME32_4 = 668265263U;+static const U32 PRIME32_5 = 374761393U;++static U32 XXH32_round(U32 seed, U32 input)+{+ seed += input * PRIME32_2;+ seed = XXH_rotl32(seed, 13);+ seed *= PRIME32_1;+ return seed;+}++/* mix all bits */+static U32 XXH32_avalanche(U32 h32)+{+ h32 ^= h32 >> 15;+ h32 *= PRIME32_2;+ h32 ^= h32 >> 13;+ h32 *= PRIME32_3;+ h32 ^= h32 >> 16;+ return(h32);+}++#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)++static U32+XXH32_finalize(U32 h32, const void* ptr, size_t len,+ XXH_endianess endian, XXH_alignment align)++{+ const BYTE* p = (const BYTE*)ptr;++#define PROCESS1 \+ h32 += (*p++) * PRIME32_5; \+ h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;++#define PROCESS4 \+ h32 += XXH_get32bits(p) * PRIME32_3; \+ p+=4; \+ h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;++ switch(len&15) /* or switch(bEnd - p) */+ {+ case 12: PROCESS4;+ /* fallthrough */+ case 8: PROCESS4;+ /* fallthrough */+ case 4: PROCESS4;+ return XXH32_avalanche(h32);++ case 13: PROCESS4;+ /* fallthrough */+ case 9: PROCESS4;+ /* fallthrough */+ case 5: PROCESS4;+ PROCESS1;+ return XXH32_avalanche(h32);++ case 14: PROCESS4;+ /* fallthrough */+ case 10: PROCESS4;+ /* fallthrough */+ case 6: PROCESS4;+ PROCESS1;+ PROCESS1;+ return XXH32_avalanche(h32);++ case 15: PROCESS4;+ /* fallthrough */+ case 11: PROCESS4;+ /* fallthrough */+ case 7: PROCESS4;+ /* fallthrough */+ case 3: PROCESS1;+ /* fallthrough */+ case 2: PROCESS1;+ /* fallthrough */+ case 1: PROCESS1;+ /* fallthrough */+ case 0: return XXH32_avalanche(h32);+ }+ assert(0);+ return h32; /* reaching this point is deemed impossible */+}+++FORCE_INLINE U32+XXH32_endian_align(const void* input, size_t len, U32 seed,+ XXH_endianess endian, XXH_alignment align)+{+ const BYTE* p = (const BYTE*)input;+ const BYTE* bEnd = p + len;+ U32 h32;++#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)+ if (p==NULL) {+ len=0;+ bEnd=p=(const BYTE*)(size_t)16;+ }+#endif++ if (len>=16) {+ const BYTE* const limit = bEnd - 15;+ U32 v1 = seed + PRIME32_1 + PRIME32_2;+ U32 v2 = seed + PRIME32_2;+ U32 v3 = seed + 0;+ U32 v4 = seed - PRIME32_1;++ do {+ v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;+ v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;+ v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;+ v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;+ } while (p < limit);++ h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7)+ + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);+ } else {+ h32 = seed + PRIME32_5;+ }++ h32 += (U32)len;++ return XXH32_finalize(h32, p, len&15, endian, align);+}+++XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)+{+#if 0+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */+ XXH32_state_t state;+ XXH32_reset(&state, seed);+ XXH32_update(&state, input, len);+ return XXH32_digest(&state);+#else+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;++ if (XXH_FORCE_ALIGN_CHECK) {+ if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)+ return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);+ else+ return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);+ } }++ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)+ return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);+ else+ return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);+#endif+}++++/*====== Hash streaming ======*/++XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)+{+ return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));+}+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)+{+ XXH_free(statePtr);+ return XXH_OK;+}++XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)+{+ memcpy(dstState, srcState, sizeof(*dstState));+}++XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)+{+ XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */+ memset(&state, 0, sizeof(state));+ state.v1 = seed + PRIME32_1 + PRIME32_2;+ state.v2 = seed + PRIME32_2;+ state.v3 = seed + 0;+ state.v4 = seed - PRIME32_1;+ /* do not write into reserved, planned to be removed in a future version */+ memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));+ return XXH_OK;+}+++FORCE_INLINE XXH_errorcode+XXH32_update_endian(XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)+{+ if (input==NULL)+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)+ return XXH_OK;+#else+ return XXH_ERROR;+#endif++ { const BYTE* p = (const BYTE*)input;+ const BYTE* const bEnd = p + len;++ state->total_len_32 += (unsigned)len;+ state->large_len |= (len>=16) | (state->total_len_32>=16);++ if (state->memsize + len < 16) { /* fill in tmp buffer */+ XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);+ state->memsize += (unsigned)len;+ return XXH_OK;+ }++ if (state->memsize) { /* some data left from previous update */+ XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);+ { const U32* p32 = state->mem32;+ state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;+ state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;+ state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;+ state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian));+ }+ p += 16-state->memsize;+ state->memsize = 0;+ }++ if (p <= bEnd-16) {+ const BYTE* const limit = bEnd - 16;+ U32 v1 = state->v1;+ U32 v2 = state->v2;+ U32 v3 = state->v3;+ U32 v4 = state->v4;++ do {+ v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;+ v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;+ v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;+ v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;+ } while (p<=limit);++ state->v1 = v1;+ state->v2 = v2;+ state->v3 = v3;+ state->v4 = v4;+ }++ if (p < bEnd) {+ XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));+ state->memsize = (unsigned)(bEnd-p);+ }+ }++ return XXH_OK;+}+++XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)+{+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;++ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)+ return XXH32_update_endian(state_in, input, len, XXH_littleEndian);+ else+ return XXH32_update_endian(state_in, input, len, XXH_bigEndian);+}+++FORCE_INLINE U32+XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)+{+ U32 h32;++ if (state->large_len) {+ h32 = XXH_rotl32(state->v1, 1)+ + XXH_rotl32(state->v2, 7)+ + XXH_rotl32(state->v3, 12)+ + XXH_rotl32(state->v4, 18);+ } else {+ h32 = state->v3 /* == seed */ + PRIME32_5;+ }++ h32 += state->total_len_32;++ return XXH32_finalize(h32, state->mem32, state->memsize, endian, XXH_aligned);+}+++XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)+{+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;++ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)+ return XXH32_digest_endian(state_in, XXH_littleEndian);+ else+ return XXH32_digest_endian(state_in, XXH_bigEndian);+}+++/*====== Canonical representation ======*/++/*! Default XXH result types are basic unsigned 32 and 64 bits.+* The canonical representation follows human-readable write convention, aka big-endian (large digits first).+* These functions allow transformation of hash result into and from its canonical format.+* This way, hash values can be written into a file or buffer, remaining comparable across different systems.+*/++XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)+{+ XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);+ memcpy(dst, &hash, sizeof(*dst));+}++XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)+{+ return XXH_readBE32(src);+}+++#ifndef XXH_NO_LONG_LONG++/* *******************************************************************+* 64-bit hash functions+*********************************************************************/++/*====== Memory access ======*/++#ifndef MEM_MODULE+# define MEM_MODULE+# if !defined (__VMS) \+ && (defined (__cplusplus) \+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )+# include <stdint.h>+ typedef uint64_t U64;+# else+ /* if compiler doesn't support unsigned long long, replace by another 64-bit type */+ typedef unsigned long long U64;+# endif+#endif+++#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))++/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */+static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }++#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))++/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */+/* currently only defined for gcc and icc */+typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64;+static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; }++#else++/* portable and safe solution. Generally efficient.+ * see : http://stackoverflow.com/a/32095106/646947+ */++static U64 XXH_read64(const void* memPtr)+{+ U64 val;+ memcpy(&val, memPtr, sizeof(val));+ return val;+}++#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */++#if defined(_MSC_VER) /* Visual Studio */+# define XXH_swap64 _byteswap_uint64+#elif XXH_GCC_VERSION >= 403+# define XXH_swap64 __builtin_bswap64+#else+static U64 XXH_swap64 (U64 x)+{+ return ((x << 56) & 0xff00000000000000ULL) |+ ((x << 40) & 0x00ff000000000000ULL) |+ ((x << 24) & 0x0000ff0000000000ULL) |+ ((x << 8) & 0x000000ff00000000ULL) |+ ((x >> 8) & 0x00000000ff000000ULL) |+ ((x >> 24) & 0x0000000000ff0000ULL) |+ ((x >> 40) & 0x000000000000ff00ULL) |+ ((x >> 56) & 0x00000000000000ffULL);+}+#endif++FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)+{+ if (align==XXH_unaligned)+ return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));+ else+ return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);+}++FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)+{+ return XXH_readLE64_align(ptr, endian, XXH_unaligned);+}++static U64 XXH_readBE64(const void* ptr)+{+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);+}+++/*====== xxh64 ======*/++static const U64 PRIME64_1 = 11400714785074694791ULL;+static const U64 PRIME64_2 = 14029467366897019727ULL;+static const U64 PRIME64_3 = 1609587929392839161ULL;+static const U64 PRIME64_4 = 9650029242287828579ULL;+static const U64 PRIME64_5 = 2870177450012600261ULL;++static U64 XXH64_round(U64 acc, U64 input)+{+ acc += input * PRIME64_2;+ acc = XXH_rotl64(acc, 31);+ acc *= PRIME64_1;+ return acc;+}++static U64 XXH64_mergeRound(U64 acc, U64 val)+{+ val = XXH64_round(0, val);+ acc ^= val;+ acc = acc * PRIME64_1 + PRIME64_4;+ return acc;+}++static U64 XXH64_avalanche(U64 h64)+{+ h64 ^= h64 >> 33;+ h64 *= PRIME64_2;+ h64 ^= h64 >> 29;+ h64 *= PRIME64_3;+ h64 ^= h64 >> 32;+ return h64;+}+++#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)++static U64+XXH64_finalize(U64 h64, const void* ptr, size_t len,+ XXH_endianess endian, XXH_alignment align)+{+ const BYTE* p = (const BYTE*)ptr;++#define PROCESS1_64 \+ h64 ^= (*p++) * PRIME64_5; \+ h64 = XXH_rotl64(h64, 11) * PRIME64_1;++#define PROCESS4_64 \+ h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \+ p+=4; \+ h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;++#define PROCESS8_64 { \+ U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \+ p+=8; \+ h64 ^= k1; \+ h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \+}++ switch(len&31) {+ case 24: PROCESS8_64;+ /* fallthrough */+ case 16: PROCESS8_64;+ /* fallthrough */+ case 8: PROCESS8_64;+ return XXH64_avalanche(h64);++ case 28: PROCESS8_64;+ /* fallthrough */+ case 20: PROCESS8_64;+ /* fallthrough */+ case 12: PROCESS8_64;+ /* fallthrough */+ case 4: PROCESS4_64;+ return XXH64_avalanche(h64);++ case 25: PROCESS8_64;+ /* fallthrough */+ case 17: PROCESS8_64;+ /* fallthrough */+ case 9: PROCESS8_64;+ PROCESS1_64;+ return XXH64_avalanche(h64);++ case 29: PROCESS8_64;+ /* fallthrough */+ case 21: PROCESS8_64;+ /* fallthrough */+ case 13: PROCESS8_64;+ /* fallthrough */+ case 5: PROCESS4_64;+ PROCESS1_64;+ return XXH64_avalanche(h64);++ case 26: PROCESS8_64;+ /* fallthrough */+ case 18: PROCESS8_64;+ /* fallthrough */+ case 10: PROCESS8_64;+ PROCESS1_64;+ PROCESS1_64;+ return XXH64_avalanche(h64);++ case 30: PROCESS8_64;+ /* fallthrough */+ case 22: PROCESS8_64;+ /* fallthrough */+ case 14: PROCESS8_64;+ /* fallthrough */+ case 6: PROCESS4_64;+ PROCESS1_64;+ PROCESS1_64;+ return XXH64_avalanche(h64);++ case 27: PROCESS8_64;+ /* fallthrough */+ case 19: PROCESS8_64;+ /* fallthrough */+ case 11: PROCESS8_64;+ PROCESS1_64;+ PROCESS1_64;+ PROCESS1_64;+ return XXH64_avalanche(h64);++ case 31: PROCESS8_64;+ /* fallthrough */+ case 23: PROCESS8_64;+ /* fallthrough */+ case 15: PROCESS8_64;+ /* fallthrough */+ case 7: PROCESS4_64;+ /* fallthrough */+ case 3: PROCESS1_64;+ /* fallthrough */+ case 2: PROCESS1_64;+ /* fallthrough */+ case 1: PROCESS1_64;+ /* fallthrough */+ case 0: return XXH64_avalanche(h64);+ }++ /* impossible to reach */+ assert(0);+ return 0; /* unreachable, but some compilers complain without it */+}++FORCE_INLINE U64+XXH64_endian_align(const void* input, size_t len, U64 seed,+ XXH_endianess endian, XXH_alignment align)+{+ const BYTE* p = (const BYTE*)input;+ const BYTE* bEnd = p + len;+ U64 h64;++#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)+ if (p==NULL) {+ len=0;+ bEnd=p=(const BYTE*)(size_t)32;+ }+#endif++ if (len>=32) {+ const BYTE* const limit = bEnd - 32;+ U64 v1 = seed + PRIME64_1 + PRIME64_2;+ U64 v2 = seed + PRIME64_2;+ U64 v3 = seed + 0;+ U64 v4 = seed - PRIME64_1;++ do {+ v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;+ v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;+ v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;+ v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;+ } while (p<=limit);++ h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);+ h64 = XXH64_mergeRound(h64, v1);+ h64 = XXH64_mergeRound(h64, v2);+ h64 = XXH64_mergeRound(h64, v3);+ h64 = XXH64_mergeRound(h64, v4);++ } else {+ h64 = seed + PRIME64_5;+ }++ h64 += (U64) len;++ return XXH64_finalize(h64, p, len, endian, align);+}+++XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)+{+#if 0+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */+ XXH64_state_t state;+ XXH64_reset(&state, seed);+ XXH64_update(&state, input, len);+ return XXH64_digest(&state);+#else+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;++ if (XXH_FORCE_ALIGN_CHECK) {+ if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)+ return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);+ else+ return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);+ } }++ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)+ return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);+ else+ return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);+#endif+}++/*====== Hash Streaming ======*/++XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)+{+ return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));+}+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)+{+ XXH_free(statePtr);+ return XXH_OK;+}++XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)+{+ memcpy(dstState, srcState, sizeof(*dstState));+}++XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)+{+ XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */+ memset(&state, 0, sizeof(state));+ state.v1 = seed + PRIME64_1 + PRIME64_2;+ state.v2 = seed + PRIME64_2;+ state.v3 = seed + 0;+ state.v4 = seed - PRIME64_1;+ /* do not write into reserved, planned to be removed in a future version */+ memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved));+ return XXH_OK;+}++FORCE_INLINE XXH_errorcode+XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)+{+ if (input==NULL)+#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1)+ return XXH_OK;+#else+ return XXH_ERROR;+#endif++ { const BYTE* p = (const BYTE*)input;+ const BYTE* const bEnd = p + len;++ state->total_len += len;++ if (state->memsize + len < 32) { /* fill in tmp buffer */+ XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);+ state->memsize += (U32)len;+ return XXH_OK;+ }++ if (state->memsize) { /* tmp buffer is full */+ XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);+ state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));+ state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));+ state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));+ state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));+ p += 32-state->memsize;+ state->memsize = 0;+ }++ if (p+32 <= bEnd) {+ const BYTE* const limit = bEnd - 32;+ U64 v1 = state->v1;+ U64 v2 = state->v2;+ U64 v3 = state->v3;+ U64 v4 = state->v4;++ do {+ v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;+ v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;+ v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;+ v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;+ } while (p<=limit);++ state->v1 = v1;+ state->v2 = v2;+ state->v3 = v3;+ state->v4 = v4;+ }++ if (p < bEnd) {+ XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));+ state->memsize = (unsigned)(bEnd-p);+ }+ }++ return XXH_OK;+}++XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)+{+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;++ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)+ return XXH64_update_endian(state_in, input, len, XXH_littleEndian);+ else+ return XXH64_update_endian(state_in, input, len, XXH_bigEndian);+}++FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)+{+ U64 h64;++ if (state->total_len >= 32) {+ U64 const v1 = state->v1;+ U64 const v2 = state->v2;+ U64 const v3 = state->v3;+ U64 const v4 = state->v4;++ h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);+ h64 = XXH64_mergeRound(h64, v1);+ h64 = XXH64_mergeRound(h64, v2);+ h64 = XXH64_mergeRound(h64, v3);+ h64 = XXH64_mergeRound(h64, v4);+ } else {+ h64 = state->v3 /*seed*/ + PRIME64_5;+ }++ h64 += (U64) state->total_len;++ return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, endian, XXH_aligned);+}++XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)+{+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;++ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)+ return XXH64_digest_endian(state_in, XXH_littleEndian);+ else+ return XXH64_digest_endian(state_in, XXH_bigEndian);+}+++/*====== Canonical representation ======*/++XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)+{+ XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);+ memcpy(dst, &hash, sizeof(*dst));+}++XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)+{+ return XXH_readBE64(src);+}++#endif /* XXH_NO_LONG_LONG */
+ lz4/lib/xxhash.h view
@@ -0,0 +1,328 @@+/*+ xxHash - Extremely Fast Hash algorithm+ Header File+ Copyright (C) 2012-2016, Yann Collet.++ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)++ Redistribution and use in source and binary forms, with or without+ modification, are permitted provided that the following conditions are+ met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.+ * Redistributions in binary form must reproduce the above+ copyright notice, this list of conditions and the following disclaimer+ in the documentation and/or other materials provided with the+ distribution.++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.++ You can contact the author at :+ - xxHash source repository : https://github.com/Cyan4973/xxHash+*/++/* Notice extracted from xxHash homepage :++xxHash is an extremely fast Hash algorithm, running at RAM speed limits.+It also successfully passes all tests from the SMHasher suite.++Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)++Name Speed Q.Score Author+xxHash 5.4 GB/s 10+CrapWow 3.2 GB/s 2 Andrew+MumurHash 3a 2.7 GB/s 10 Austin Appleby+SpookyHash 2.0 GB/s 10 Bob Jenkins+SBox 1.4 GB/s 9 Bret Mulvey+Lookup3 1.2 GB/s 9 Bob Jenkins+SuperFastHash 1.2 GB/s 1 Paul Hsieh+CityHash64 1.05 GB/s 10 Pike & Alakuijala+FNV 0.55 GB/s 5 Fowler, Noll, Vo+CRC32 0.43 GB/s 9+MD5-32 0.33 GB/s 10 Ronald L. Rivest+SHA1-32 0.28 GB/s 10++Q.Score is a measure of quality of the hash function.+It depends on successfully passing SMHasher test set.+10 is a perfect score.++A 64-bit version, named XXH64, is available since r35.+It offers much better speed, but for 64-bit applications only.+Name Speed on 64 bits Speed on 32 bits+XXH64 13.8 GB/s 1.9 GB/s+XXH32 6.8 GB/s 6.0 GB/s+*/++#ifndef XXHASH_H_5627135585666179+#define XXHASH_H_5627135585666179 1++#if defined (__cplusplus)+extern "C" {+#endif+++/* ****************************+* Definitions+******************************/+#include <stddef.h> /* size_t */+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;+++/* ****************************+ * API modifier+ ******************************/+/** XXH_INLINE_ALL (and XXH_PRIVATE_API)+ * This is useful to include xxhash functions in `static` mode+ * in order to inline them, and remove their symbol from the public list.+ * Inlining can offer dramatic performance improvement on small keys.+ * Methodology :+ * #define XXH_INLINE_ALL+ * #include "xxhash.h"+ * `xxhash.c` is automatically included.+ * It's not useful to compile and link it as a separate module.+ */+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)+# ifndef XXH_STATIC_LINKING_ONLY+# define XXH_STATIC_LINKING_ONLY+# endif+# if defined(__GNUC__)+# define XXH_PUBLIC_API static __inline __attribute__((unused))+# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)+# define XXH_PUBLIC_API static inline+# elif defined(_MSC_VER)+# define XXH_PUBLIC_API static __inline+# else+ /* this version may generate warnings for unused static functions */+# define XXH_PUBLIC_API static+# endif+#else+# define XXH_PUBLIC_API /* do nothing */+#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */++/*! XXH_NAMESPACE, aka Namespace Emulation :+ *+ * If you want to include _and expose_ xxHash functions from within your own library,+ * but also want to avoid symbol collisions with other libraries which may also include xxHash,+ *+ * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library+ * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).+ *+ * Note that no change is required within the calling program as long as it includes `xxhash.h` :+ * regular symbol name will be automatically translated by this header.+ */+#ifdef XXH_NAMESPACE+# define XXH_CAT(A,B) A##B+# define XXH_NAME2(A,B) XXH_CAT(A,B)+# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)+# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)+# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)+# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)+# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)+# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)+# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)+# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)+# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)+# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)+# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)+# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)+# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)+# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)+# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)+# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)+# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)+# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)+# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)+#endif+++/* *************************************+* Version+***************************************/+#define XXH_VERSION_MAJOR 0+#define XXH_VERSION_MINOR 6+#define XXH_VERSION_RELEASE 5+#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)+XXH_PUBLIC_API unsigned XXH_versionNumber (void);+++/*-**********************************************************************+* 32-bit hash+************************************************************************/+typedef unsigned int XXH32_hash_t;++/*! XXH32() :+ Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input".+ The memory between input & input+length must be valid (allocated and read-accessible).+ "seed" can be used to alter the result predictably.+ Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);++/*====== Streaming ======*/+typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);++XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed);+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);+XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);++/*+ * Streaming functions generate the xxHash of an input provided in multiple segments.+ * Note that, for small input, they are slower than single-call functions, due to state management.+ * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.+ *+ * XXH state must first be allocated, using XXH*_createState() .+ *+ * Start a new hash by initializing state with a seed, using XXH*_reset().+ *+ * Then, feed the hash state by calling XXH*_update() as many times as necessary.+ * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.+ *+ * Finally, a hash value can be produced anytime, by using XXH*_digest().+ * This function returns the nn-bits hash as an int or long long.+ *+ * It's still possible to continue inserting input into the hash state after a digest,+ * and generate some new hashes later on, by calling again XXH*_digest().+ *+ * When done, free XXH state space if it was allocated dynamically.+ */++/*====== Canonical representation ======*/++typedef struct { unsigned char digest[4]; } XXH32_canonical_t;+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);++/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.+ * The canonical representation uses human-readable write convention, aka big-endian (large digits first).+ * These functions allow transformation of hash result into and from its canonical format.+ * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.+ */+++#ifndef XXH_NO_LONG_LONG+/*-**********************************************************************+* 64-bit hash+************************************************************************/+typedef unsigned long long XXH64_hash_t;++/*! XXH64() :+ Calculate the 64-bit hash of sequence of length "len" stored at memory address "input".+ "seed" can be used to alter the result predictably.+ This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark).+*/+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);++/*====== Streaming ======*/+typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);++XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);+XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);++/*====== Canonical representation ======*/+typedef struct { unsigned char digest[8]; } XXH64_canonical_t;+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);+#endif /* XXH_NO_LONG_LONG */++++#ifdef XXH_STATIC_LINKING_ONLY++/* ================================================================================================+ This section contains declarations which are not guaranteed to remain stable.+ They may change in future versions, becoming incompatible with a different version of the library.+ These declarations should only be used with static linking.+ Never use them in association with dynamic linking !+=================================================================================================== */++/* These definitions are only present to allow+ * static allocation of XXH state, on stack or in a struct for example.+ * Never **ever** use members directly. */++#if !defined (__VMS) \+ && (defined (__cplusplus) \+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )+# include <stdint.h>++struct XXH32_state_s {+ uint32_t total_len_32;+ uint32_t large_len;+ uint32_t v1;+ uint32_t v2;+ uint32_t v3;+ uint32_t v4;+ uint32_t mem32[4];+ uint32_t memsize;+ uint32_t reserved; /* never read nor write, might be removed in a future version */+}; /* typedef'd to XXH32_state_t */++struct XXH64_state_s {+ uint64_t total_len;+ uint64_t v1;+ uint64_t v2;+ uint64_t v3;+ uint64_t v4;+ uint64_t mem64[4];+ uint32_t memsize;+ uint32_t reserved[2]; /* never read nor write, might be removed in a future version */+}; /* typedef'd to XXH64_state_t */++# else++struct XXH32_state_s {+ unsigned total_len_32;+ unsigned large_len;+ unsigned v1;+ unsigned v2;+ unsigned v3;+ unsigned v4;+ unsigned mem32[4];+ unsigned memsize;+ unsigned reserved; /* never read nor write, might be removed in a future version */+}; /* typedef'd to XXH32_state_t */++# ifndef XXH_NO_LONG_LONG /* remove 64-bit support */+struct XXH64_state_s {+ unsigned long long total_len;+ unsigned long long v1;+ unsigned long long v2;+ unsigned long long v3;+ unsigned long long v4;+ unsigned long long mem64[4];+ unsigned memsize;+ unsigned reserved[2]; /* never read nor write, might be removed in a future version */+}; /* typedef'd to XXH64_state_t */+# endif++# endif+++#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)+# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */+#endif++#endif /* XXH_STATIC_LINKING_ONLY */+++#if defined (__cplusplus)+}+#endif++#endif /* XXHASH_H_5627135585666179 */
+ src/Codec/Compression/LZ4/CTypes.hsc view
@@ -0,0 +1,239 @@+{-# LANGUAGE NamedFieldPuns #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE TemplateHaskell #-}++module Codec.Compression.LZ4.CTypes+ ( Lz4FrameException(..)+ , BlockSizeID(..)+ , BlockMode(..)+ , ContentChecksum(..)+ , BlockChecksum(..)+ , FrameType(..)+ , FrameInfo(..)+ , Preferences(..)+ , LZ4F_cctx+ , LZ4F_dctx+ , lz4FrameTypesTable+ ) where++import Control.Exception (Exception, throwIO)+import Data.Map (Map)+import qualified Data.Map as Map+import Data.Typeable (Typeable)+import Data.Word (Word32, Word64)+import Foreign.Marshal.Utils (fillBytes)+import Foreign.Ptr (Ptr, castPtr)+import Foreign.Storable (Storable(..), poke)+import qualified Language.C.Types as C+import qualified Language.Haskell.TH as TH++#include "lz4frame.h"+++data Lz4FrameException = Lz4FormatException String+ deriving (Eq, Ord, Show, Typeable)++instance Exception Lz4FrameException+++data BlockSizeID+ = LZ4F_default+ | LZ4F_max64KB+ | LZ4F_max256KB+ | LZ4F_max1MB+ | LZ4F_max4MB+ deriving (Eq, Ord, Show)++instance Storable BlockSizeID where+ sizeOf _ = #{size LZ4F_blockSizeID_t}+ alignment _ = #{alignment LZ4F_blockSizeID_t}+ peek p = do+ n <- peek (castPtr p :: Ptr #{type LZ4F_blockSizeID_t})+ case n of+ #{const LZ4F_default} -> return LZ4F_default+ #{const LZ4F_max64KB} -> return LZ4F_max64KB+ #{const LZ4F_max256KB} -> return LZ4F_max256KB+ #{const LZ4F_max1MB} -> return LZ4F_max1MB+ #{const LZ4F_max4MB} -> return LZ4F_max4MB+ _ -> throwIO $ Lz4FormatException $ "lz4 instance Storable BlockSizeID: encountered unknown LZ4F_blockSizeID_t: " ++ show n+ poke p i = poke (castPtr p :: Ptr #{type LZ4F_blockSizeID_t}) $ case i of+ LZ4F_default -> #{const LZ4F_default}+ LZ4F_max64KB -> #{const LZ4F_max64KB}+ LZ4F_max256KB -> #{const LZ4F_max256KB}+ LZ4F_max1MB -> #{const LZ4F_max1MB}+ LZ4F_max4MB -> #{const LZ4F_max4MB}+++data BlockMode+ = LZ4F_blockLinked+ | LZ4F_blockIndependent+ deriving (Eq, Ord, Show)++instance Storable BlockMode where+ sizeOf _ = #{size LZ4F_blockMode_t}+ alignment _ = #{alignment LZ4F_blockMode_t}+ peek p = do+ n <- peek (castPtr p :: Ptr #{type LZ4F_blockMode_t})+ case n of+ #{const LZ4F_blockLinked } -> return LZ4F_blockLinked+ #{const LZ4F_blockIndependent } -> return LZ4F_blockIndependent+ _ -> throwIO $ Lz4FormatException $ "lz4 instance Storable BlockMode: encountered unknown LZ4F_blockMode_t: " ++ show n+ poke p mode = poke (castPtr p :: Ptr #{type LZ4F_blockMode_t}) $ case mode of+ LZ4F_blockLinked -> #{const LZ4F_blockLinked}+ LZ4F_blockIndependent -> #{const LZ4F_blockIndependent}+++data ContentChecksum+ = LZ4F_noContentChecksum+ | LZ4F_contentChecksumEnabled+ deriving (Eq, Ord, Show)++instance Storable ContentChecksum where+ sizeOf _ = #{size LZ4F_contentChecksum_t}+ alignment _ = #{alignment LZ4F_contentChecksum_t}+ peek p = do+ n <- peek (castPtr p :: Ptr #{type LZ4F_contentChecksum_t})+ case n of+ #{const LZ4F_noContentChecksum } -> return LZ4F_noContentChecksum+ #{const LZ4F_contentChecksumEnabled } -> return LZ4F_contentChecksumEnabled+ _ -> throwIO $ Lz4FormatException $ "lz4 instance Storable ContentChecksum: encountered unknown LZ4F_contentChecksum_t: " ++ show n+ poke p mode = poke (castPtr p :: Ptr #{type LZ4F_contentChecksum_t}) $ case mode of+ LZ4F_noContentChecksum -> #{const LZ4F_noContentChecksum}+ LZ4F_contentChecksumEnabled -> #{const LZ4F_contentChecksumEnabled}+++data BlockChecksum+ = LZ4F_noBlockChecksum+ | LZ4F_blockChecksumEnabled+ deriving (Eq, Ord, Show)++instance Storable BlockChecksum where+ sizeOf _ = #{size LZ4F_blockChecksum_t}+ alignment _ = #{alignment LZ4F_blockChecksum_t}+ peek p = do+ n <- peek (castPtr p :: Ptr #{type LZ4F_blockChecksum_t})+ case n of+ #{const LZ4F_noBlockChecksum } -> return LZ4F_noBlockChecksum+ #{const LZ4F_blockChecksumEnabled } -> return LZ4F_blockChecksumEnabled+ _ -> throwIO $ Lz4FormatException $ "lz4 instance Storable BlockChecksum: encountered unknown LZ4F_blockChecksum_t: " ++ show n+ poke p mode = poke (castPtr p :: Ptr #{type LZ4F_blockChecksum_t}) $ case mode of+ LZ4F_noBlockChecksum -> #{const LZ4F_noBlockChecksum}+ LZ4F_blockChecksumEnabled -> #{const LZ4F_blockChecksumEnabled}+++data FrameType+ = LZ4F_frame+ | LZ4F_skippableFrame+ deriving (Eq, Ord, Show)++instance Storable FrameType where+ sizeOf _ = #{size LZ4F_frameType_t}+ alignment _ = #{alignment LZ4F_frameType_t}+ peek p = do+ n <- peek (castPtr p :: Ptr #{type LZ4F_frameType_t})+ case n of+ #{const LZ4F_frame } -> return LZ4F_frame+ #{const LZ4F_skippableFrame } -> return LZ4F_skippableFrame+ _ -> throwIO $ Lz4FormatException $ "lz4 instance Storable FrameType: encountered unknown LZ4F_frameType_t: " ++ show n+ poke p mode = poke (castPtr p :: Ptr #{type LZ4F_frameType_t}) $ case mode of+ LZ4F_frame -> #{const LZ4F_frame}+ LZ4F_skippableFrame -> #{const LZ4F_skippableFrame}+++data FrameInfo = FrameInfo+ { blockSizeID :: BlockSizeID+ , blockMode :: BlockMode+ , contentChecksumFlag :: ContentChecksum+ , frameType :: FrameType+ , contentSize :: Word64+ , dictID :: Word32 -- ^ @unsigned int@ in @lz4frame.h@, which can be 16 or 32 bits; AFAIK GHC does not run on archs where it is 16-bit, so there's a compile-time check for it.+ , blockChecksumFlag :: BlockChecksum+ }++-- See comment on `dictID`.+$(if #{size unsigned} /= (4 :: Int)+ then error "sizeof(unsigned) is not 4 (32-bits), the code is not written for this"+ else pure []+ )++instance Storable FrameInfo where+ sizeOf _ = #{size LZ4F_frameInfo_t}+ alignment _ = #{alignment LZ4F_frameInfo_t}+ peek p = do+ blockSizeID <- #{peek LZ4F_frameInfo_t, blockSizeID} p+ blockMode <- #{peek LZ4F_frameInfo_t, blockMode} p+ contentChecksumFlag <- #{peek LZ4F_frameInfo_t, contentChecksumFlag} p+ frameType <- #{peek LZ4F_frameInfo_t, frameType} p+ contentSize <- #{peek LZ4F_frameInfo_t, contentSize} p+ dictID <- #{peek LZ4F_frameInfo_t, dictID} p+ blockChecksumFlag <- #{peek LZ4F_frameInfo_t, blockChecksumFlag} p+ return $ FrameInfo+ { blockSizeID+ , blockMode+ , contentChecksumFlag+ , frameType+ , contentSize+ , dictID+ , blockChecksumFlag+ }+ poke p frameInfo = do+ #{poke LZ4F_frameInfo_t, blockSizeID} p $ blockSizeID frameInfo+ #{poke LZ4F_frameInfo_t, blockMode} p $ blockMode frameInfo+ #{poke LZ4F_frameInfo_t, contentChecksumFlag} p $ contentChecksumFlag frameInfo+ #{poke LZ4F_frameInfo_t, frameType} p $ frameType frameInfo+ #{poke LZ4F_frameInfo_t, contentSize} p $ contentSize frameInfo+ -- These were reserved fields once; versions of `lz4frame.h` older+ -- than v1.8.0 will not have them.+ #{poke LZ4F_frameInfo_t, dictID} p $ dictID frameInfo+ #{poke LZ4F_frameInfo_t, blockChecksumFlag} p $ blockChecksumFlag frameInfo+++data Preferences = Preferences+ { frameInfo :: FrameInfo+ , compressionLevel :: Int+ , autoFlush :: Bool+ , favorDecSpeed :: Bool+ }++instance Storable Preferences where+ sizeOf _ = #{size LZ4F_preferences_t}+ alignment _ = #{alignment LZ4F_preferences_t}+ peek p = do+ frameInfo <- #{peek LZ4F_preferences_t, frameInfo} p+ compressionLevel <- #{peek LZ4F_preferences_t, compressionLevel} p+ autoFlush <- #{peek LZ4F_preferences_t, autoFlush} p+ favorDecSpeed <- #{peek LZ4F_preferences_t, favorDecSpeed} p+ return $ Preferences+ { frameInfo+ , compressionLevel+ , autoFlush+ , favorDecSpeed+ }+ poke p preferences = do+ fillBytes p 0 #{size LZ4F_preferences_t} -- set reserved fields to 0 as lz4frame.h requires+ #{poke LZ4F_preferences_t, frameInfo} p $ frameInfo preferences+ #{poke LZ4F_preferences_t, compressionLevel} p $ compressionLevel preferences+ #{poke LZ4F_preferences_t, autoFlush} p $ autoFlush preferences+ #{poke LZ4F_preferences_t, favorDecSpeed} p $ favorDecSpeed preferences -- since lz4 v1.8.2+ -- reserved uint field here, see lz4frame.h+ -- reserved uint field here, see lz4frame.h+ -- reserved uint field here, see lz4frame.h+++data LZ4F_cctx+data LZ4F_dctx+++lz4FrameTypesTable :: Map C.TypeSpecifier TH.TypeQ+lz4FrameTypesTable = Map.fromList+ [ (C.TypeName "LZ4F_cctx", [t| LZ4F_cctx |])+ , (C.TypeName "LZ4F_dctx", [t| LZ4F_dctx |])+ , (C.TypeName "LZ4F_blockSizeID_t", [t| BlockSizeID |])+ , (C.TypeName "LZ4F_blockMode_t", [t| BlockMode |])+ , (C.TypeName "LZ4F_contentChecksum_t", [t| ContentChecksum |])+ , (C.TypeName "LZ4F_blockChecksum_t", [t| BlockChecksum |])+ , (C.TypeName "LZ4F_frameInfo_t", [t| FrameInfo |])+ , (C.TypeName "LZ4F_frameType_t", [t| FrameType |])+ , (C.TypeName "LZ4F_preferences_t", [t| Preferences |])+ ]
+ src/Codec/Compression/LZ4/Conduit.hsc view
@@ -0,0 +1,692 @@+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE MultiWayIf #-}+{-# LANGUAGE PartialTypeSignatures #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TemplateHaskell #-}+{-# OPTIONS_GHC -fno-warn-partial-type-signatures #-}++-- | TODO: Implement:+--+-- * Block checksumming+-- * Dictionary support+module Codec.Compression.LZ4.Conduit+ ( Lz4FrameException(..)+ , BlockSizeID(..)+ , BlockMode(..)+ , ContentChecksum(..)+ , BlockChecksum(..)+ , FrameType(..)+ , FrameInfo(..)+ , Preferences(..)++ , lz4DefaultPreferences++ , compress+ , compressYieldImmediately+ , compressWithOutBufferSize++ , decompress++ , bsChunksOf++ -- * Internals+ , Lz4FrameCompressionContext(..)+ , ScopedLz4FrameCompressionContext(..)+ , ScopedLz4FramePreferencesPtr(..)+ , Lz4FramePreferencesPtr(..)+ , Lz4FrameDecompressionContext(..)+ , lz4fCreatePreferences+ , lz4fCreateCompressonContext+ , lz4fCreateDecompressionContext+ , withScopedLz4fPreferences+ , withScopedLz4fCompressionContext+ ) where++import UnliftIO.Exception (throwString, bracket)+import Control.Monad (foldM, when)+import Control.Monad.IO.Unlift (MonadUnliftIO)+import Control.Monad.IO.Class (liftIO)+import Control.Monad.Trans.Resource (MonadResource)+import Data.Bits (testBit)+import Data.ByteString (ByteString, packCStringLen)+import Data.ByteString.Unsafe (unsafePackCString, unsafeUseAsCStringLen)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Char8 as BS8+import qualified Data.ByteString.Lazy as BSL+import Data.Conduit+import qualified Data.Conduit.Binary as CB+import Data.Monoid ((<>))+import Foreign.C.Types (CChar, CSize)+import Foreign.ForeignPtr (ForeignPtr, addForeignPtrFinalizer, mallocForeignPtr, mallocForeignPtrBytes, finalizeForeignPtr, withForeignPtr)+import Foreign.Marshal.Alloc (alloca, allocaBytes, malloc, free)+import Foreign.Marshal.Array (mallocArray, reallocArray)+import Foreign.Marshal.Utils (with, new)+import Foreign.Ptr (Ptr, nullPtr, FunPtr, plusPtr)+import Foreign.Storable (Storable(..), poke)+import GHC.Stack (HasCallStack)+import qualified Language.C.Inline as C+import qualified Language.C.Inline.Context as C+import qualified Language.C.Inline.Unsafe as CUnsafe+import Text.RawString.QQ++import Codec.Compression.LZ4.CTypes (LZ4F_cctx, LZ4F_dctx, lz4FrameTypesTable, Lz4FrameException(..), BlockSizeID(..), BlockMode(..), ContentChecksum(..), BlockChecksum(..), FrameType(..), FrameInfo(..), Preferences(..))++#include "lz4frame.h"++C.context (C.baseCtx <> C.fptrCtx <> mempty { C.ctxTypesTable = lz4FrameTypesTable })++C.include "<lz4frame.h>"+C.include "<stdlib.h>"+C.include "<stdio.h>"+++newtype Lz4FrameCompressionContext = Lz4FrameCompressionContext { unLz4FrameCompressionContext :: ForeignPtr (Ptr LZ4F_cctx) }+ deriving (Eq, Ord, Show)++newtype ScopedLz4FrameCompressionContext = ScopedLz4FrameCompressionContext { unScopedLz4FrameCompressionContext :: Ptr LZ4F_cctx }+ deriving (Eq, Ord, Show)++newtype ScopedLz4FramePreferencesPtr = ScopedLz4FramePreferencesPtr { unScopedLz4FramePreferencesPtr :: Ptr Preferences }+ deriving (Eq, Ord, Show)++newtype Lz4FramePreferencesPtr = Lz4FramePreferencesPtr { unLz4FramePreferencesPtr :: ForeignPtr Preferences }+ deriving (Eq, Ord, Show)++newtype Lz4FrameDecompressionContext = Lz4FrameDecompressionContext { unLz4FrameDecompressionContext :: ForeignPtr (Ptr LZ4F_dctx) }+ deriving (Eq, Ord, Show)+++handleLz4Error :: (HasCallStack, MonadUnliftIO m) => IO CSize -> m CSize+handleLz4Error f = do+ ret <- liftIO f+ -- We use an unsafe foreign call here so that it's fast (it cannot block).+ staticErrMsgPtr <- liftIO [CUnsafe.exp| const char * {+ LZ4F_isError($(size_t ret))+ ? LZ4F_getErrorName($(size_t ret))+ : NULL+ } |]+ if staticErrMsgPtr == nullPtr+ then return ret+ else do+ -- LZ4F error strings are static memory so they don't need to be GC'd,+ -- so we should use `unsafePackCString` here.+ errMsgBs <- liftIO $ unsafePackCString staticErrMsgPtr+ throwString ("lz4frame error: " ++ BS8.unpack errMsgBs)+++C.verbatim [r|+void haskell_lz4_freeCompressionContext(LZ4F_cctx** ctxPtr)+{+ // We know ctxPtr can be dereferenced because it was created with+ // mallocForeignPtr, so it is always pointing to something valid+ // during the lifetime of the ForeignPtr (and this function is+ // a finalizer function, which is called only at the very end+ // inside this lifetime).+ LZ4F_cctx* ctx = *ctxPtr;+ // See note [Initialize LZ4 context pointer to NULL]:+ // If ctx is null, we never made a successful call to+ // LZ4F_createCompressionContext().+ // Note at the time of writing the implementation of+ // LZ4F_createCompressionContext() handles null pointers gracefully,+ // but that is an undocumented implementation detail so we don't+ // rely on it here.+ if (ctx != NULL)+ {+ size_t err = LZ4F_freeCompressionContext(ctx);+ if (LZ4F_isError(err))+ {+ fprintf(stderr, "LZ4F_freeCompressionContext failed: %s\n", LZ4F_getErrorName(err));+ exit(1);+ }+ }+}+|]++foreign import ccall "&haskell_lz4_freeCompressionContext" haskell_lz4_freeCompressionContext :: FunPtr (Ptr (Ptr LZ4F_cctx) -> IO ())+++allocateLz4fScopedCompressionContext :: IO ScopedLz4FrameCompressionContext+allocateLz4fScopedCompressionContext = do+ alloca $ \(ctxPtrPtr :: Ptr (Ptr LZ4F_cctx)) -> do+ _ <- handleLz4Error [C.block| size_t {+ LZ4F_cctx** ctxPtr = $(LZ4F_cctx** ctxPtrPtr);+ LZ4F_errorCode_t err = LZ4F_createCompressionContext(ctxPtr, LZ4F_VERSION);+ return err;+ } |]+ ctxPtr <- peek ctxPtrPtr+ return (ScopedLz4FrameCompressionContext ctxPtr)+++freeLz4ScopedCompressionContext :: ScopedLz4FrameCompressionContext -> IO ()+freeLz4ScopedCompressionContext (ScopedLz4FrameCompressionContext ctxPtr) = do+ _ <- handleLz4Error+ [C.block| size_t {+ return LZ4F_freeCompressionContext($(LZ4F_cctx* ctxPtr));+ } |]+ return ()++++-- TODO Performance:+-- Write a version of `compress` that emits ByteStrings of known+-- constant length. That will allow us to do compression in a zero-copy+-- fashion, writing compressed bytes directly into a the ByteStrings+-- (e.g using `unsafePackMallocCString` or equivalent).+-- We currently don't do that (instead, use allocaBytes + copying packCStringLen)+-- to ensure that the ByteStrings generated are as compact as possible+-- (for the case that `written < size`), since the current `compress`+-- conduit directly yields the outputs of LZ4F_compressUpdate()+-- (unless they are of 0 length when they are buffered in the context+-- tmp buffer).++-- TODO Try enabling checksums, then corrupt a bit and see if lz4c detects it.++-- TODO Add `with*` style bracketed functions for creating the+-- LZ4F_createCompressionContext and Lz4FramePreferencesPtr+-- for prompt resource release,+-- in addition to the GC'd variants below.+-- This would replace our use of `finalizeForeignPtr` in the conduit.+-- `finalizeForeignPtr` seems almost as good, but note that it+-- doesn't guarantee prompt resource release on exceptions;+-- a `with*` style function that uses `bracket` does.+-- However, it isn't clear yet which one would be faster+-- (what the cost of `mask` is compared to foreign pointer finalizers).+-- Also note that prompt freeing has side benefits,+-- such as reduced malloc() fragmentation (the closer malloc()+-- and free() are to each other, the smaller is the chance to+-- have malloc()s on top of the our malloc() in the heap,+-- thus the smaller the chance that we cannot decrease the+-- heap pointer upon free() (because "mallocs on top" render+-- heap memory unreturnable to the OS; memory fragmentation).+++-- TODO Turn the above TODO into documentation+++withScopedLz4fCompressionContext :: (HasCallStack) => (ScopedLz4FrameCompressionContext -> IO a) -> IO a+withScopedLz4fCompressionContext f =+ bracket+ allocateLz4fScopedCompressionContext+ freeLz4ScopedCompressionContext+ f+++lz4fCreateCompressonContext :: (HasCallStack) => IO Lz4FrameCompressionContext+lz4fCreateCompressonContext = do+ ctxForeignPtr :: ForeignPtr (Ptr LZ4F_cctx) <- mallocForeignPtr+ -- Note [Initialize LZ4 context pointer to NULL]:+ -- We explicitly set it to NULL so that in the finalizer we know+ -- whether there is a context to free or not.+ withForeignPtr ctxForeignPtr $ \ptr -> poke ptr nullPtr++ -- Attach finalizer *before* we call LZ4F_createCompressionContext(),+ -- to ensure there cannot be a time where the context was created+ -- but not finalizer is attached (receiving an async exception at+ -- that time would make us leak memory).+ addForeignPtrFinalizer haskell_lz4_freeCompressionContext ctxForeignPtr+ -- TODO The whole idea above is to avoid `mask`.+ -- But we should check if `addForeignPtrFinalizer` itself is actually+ -- async exception safe; if not, this is pointless.++ _ <- handleLz4Error [C.block| size_t {+ LZ4F_cctx** ctxPtr = $fptr-ptr:(LZ4F_cctx** ctxForeignPtr);+ LZ4F_errorCode_t err = LZ4F_createCompressionContext(ctxPtr, LZ4F_VERSION);+ return err;+ } |]+ return (Lz4FrameCompressionContext ctxForeignPtr)+++lz4DefaultPreferences :: Preferences+lz4DefaultPreferences =+ Preferences+ { frameInfo = FrameInfo+ { blockSizeID = LZ4F_default+ , blockMode = LZ4F_blockLinked+ , contentChecksumFlag = LZ4F_noContentChecksum+ , frameType = LZ4F_frame+ , contentSize = 0+ , dictID = 0+ , blockChecksumFlag = LZ4F_noBlockChecksum+ }+ , compressionLevel = 0+ , autoFlush = False+ , favorDecSpeed = False+ }+++newForeignPtr :: (Storable a) => a -> IO (ForeignPtr a)+newForeignPtr x = do+ fptr <- mallocForeignPtr+ withForeignPtr fptr $ \ptr -> poke ptr x+ return fptr+++lz4fCreatePreferences :: IO Lz4FramePreferencesPtr+lz4fCreatePreferences =+ Lz4FramePreferencesPtr <$> newForeignPtr lz4DefaultPreferences+++withScopedLz4fPreferences :: (HasCallStack) => (ScopedLz4FramePreferencesPtr -> IO a) -> IO a+withScopedLz4fPreferences f =+ bracket+ (ScopedLz4FramePreferencesPtr <$> new lz4DefaultPreferences)+ (\(ScopedLz4FramePreferencesPtr ptr) -> free ptr)+ f+++lz4fCompressBegin :: (HasCallStack) => ScopedLz4FrameCompressionContext -> ScopedLz4FramePreferencesPtr -> Ptr CChar -> CSize -> IO CSize+lz4fCompressBegin (ScopedLz4FrameCompressionContext ctx) (ScopedLz4FramePreferencesPtr prefsPtr) headerBuf headerBufLen = do+ headerSize <- handleLz4Error [C.block| size_t {++ LZ4F_preferences_t* lz4_preferences_ptr = $(LZ4F_preferences_t* prefsPtr);++ size_t err_or_headerSize = LZ4F_compressBegin($(LZ4F_cctx* ctx), $(char* headerBuf), $(size_t headerBufLen), lz4_preferences_ptr);+ return err_or_headerSize;+ } |]++ return headerSize+++lz4fCompressBound :: (HasCallStack) => CSize -> ScopedLz4FramePreferencesPtr -> IO CSize+lz4fCompressBound srcSize (ScopedLz4FramePreferencesPtr prefsPtr) = do+ handleLz4Error [C.block| size_t {+ size_t err_or_frame_size = LZ4F_compressBound($(size_t srcSize), $(LZ4F_preferences_t* prefsPtr));+ return err_or_frame_size;+ } |]+++lz4fCompressUpdate :: (HasCallStack) => ScopedLz4FrameCompressionContext -> Ptr CChar -> CSize -> Ptr CChar -> CSize -> IO CSize+lz4fCompressUpdate (ScopedLz4FrameCompressionContext ctx) destBuf destBufLen srcBuf srcBufLen = do+ -- TODO allow passing in cOptPtr instead of NULL.+ written <- handleLz4Error [C.block| size_t {+ size_t err_or_written = LZ4F_compressUpdate($(LZ4F_cctx* ctx), $(char* destBuf), $(size_t destBufLen), $(char* srcBuf), $(size_t srcBufLen), NULL);+ return err_or_written;+ } |]+ return written+++lz4fCompressEnd :: (HasCallStack) => ScopedLz4FrameCompressionContext -> Ptr CChar -> CSize -> IO CSize+lz4fCompressEnd (ScopedLz4FrameCompressionContext ctx) footerBuf footerBufLen = do+ -- TODO allow passing in cOptPtr instead of NULL.+ footerWritten <- handleLz4Error [C.block| size_t {+ size_t err_or_footerWritten = LZ4F_compressEnd($(LZ4F_cctx* ctx), $(char* footerBuf), $(size_t footerBufLen), NULL);+ return err_or_footerWritten;+ } |]+ return footerWritten+++-- Note [Single call to LZ4F_compressUpdate() can create multiple blocks]+-- A single call to LZ4F_compressUpdate() can create multiple blocks,+-- and handles buffers > 32-bit sizes; see:+-- https://github.com/lz4/lz4/blob/52cac9a97342641315c76cfb861206d6acd631a8/lib/lz4frame.c#L601+-- So we don't need to loop around LZ4F_compressUpdate() to compress+-- an arbitrarily large amount of input data, as long as the destination+-- buffer is large enough.+++compress :: (MonadUnliftIO m, MonadResource m) => ConduitT ByteString ByteString m ()+compress = compressWithOutBufferSize 0+++withLz4CtxAndPrefsConduit ::+ (MonadUnliftIO m, MonadResource m)+ => ((ScopedLz4FrameCompressionContext, ScopedLz4FramePreferencesPtr) -> ConduitT i o m r)+ -> ConduitT i o m r+withLz4CtxAndPrefsConduit f = bracketP+ (do+ ctx <- allocateLz4fScopedCompressionContext+ prefPtr <- new lz4DefaultPreferences+ return (ctx, ScopedLz4FramePreferencesPtr prefPtr)+ )+ (\(ctx, ScopedLz4FramePreferencesPtr prefPtr) -> do+ freeLz4ScopedCompressionContext ctx+ free prefPtr+ )+ f+++-- | Compresses the incoming stream of ByteStrings with the lz4 frame format.+--+-- Yields every LZ4 output as a ByteString as soon as the lz4 frame+-- library produces it.+--+-- Note that this does not imply ZL4 frame autoFlush (which affects+-- when the lz4 frame library produces outputs).+compressYieldImmediately :: (MonadUnliftIO m, MonadResource m) => ConduitT ByteString ByteString m ()+compressYieldImmediately =+ withLz4CtxAndPrefsConduit $ \(ctx, prefs) -> do+ let _LZ4F_HEADER_SIZE_MAX = #{const LZ4F_HEADER_SIZE_MAX}++ -- Header++ headerBs <- liftIO $ allocaBytes (fromIntegral _LZ4F_HEADER_SIZE_MAX) $ \headerBuf -> do+ headerSize <- lz4fCompressBegin ctx prefs headerBuf _LZ4F_HEADER_SIZE_MAX+ packCStringLen (headerBuf, fromIntegral headerSize)++ yield headerBs++ -- Chunks++ awaitForever $ \bs -> do+++ m'outBs <- liftIO $ unsafeUseAsCStringLen bs $ \(bsPtr, bsLen) -> do+ let bsLenSize = fromIntegral bsLen++ size <- lz4fCompressBound bsLenSize prefs++ -- TODO Performance: Check if reusing this buffer obtained with `allocaBytes`+ -- makes it faster.+ -- LZ4F_compressBound() always returns a number > the block size (e.g. 256K),+ -- even when the input size passed to it is just a few bytes.+ -- As a result, we allocate at least a full block size each time+ -- (and `allocaBytes` calls `malloc()`), but not using most of it.+ -- Worse, with autoflush=0, most small inputs go into the context buffer,+ -- in which case the `allocaBytes` is completely wasted.+ -- This could be avoided by keeping the last `allocaBytes` buffer around,+ -- and reusing it if it is big enough for the number returned by+ -- LZ4F_compressUpdate() next time.+ -- That should avoid most allocations in the case that we `await` lots of+ -- small ByteStrings.+ m'outBs <- liftIO $ allocaBytes (fromIntegral size) $ \buf -> do++ -- See note [Single call to LZ4F_compressUpdate() can create multiple blocks].+ written <- lz4fCompressUpdate ctx buf size bsPtr bsLenSize++ if written == 0 -- everything fit into the context buffer, no new compressed data was emitted+ then return Nothing+ else Just <$> packCStringLen (buf, fromIntegral written)++ return m'outBs++ case m'outBs of+ Nothing -> return ()+ Just outBs -> yield outBs++ -- Footer++ -- Passing srcSize==0 provides bound for LZ4F_compressEnd(),+ -- see docs of LZ4F_compressBound() for that.+ footerSize <- liftIO $ lz4fCompressBound 0 prefs++ footerBs <- liftIO $ allocaBytes (fromIntegral footerSize) $ \footerBuf -> do+ footerWritten <- lz4fCompressEnd ctx footerBuf footerSize+ packCStringLen (footerBuf, fromIntegral footerWritten)++ yield footerBs+++bsChunksOf :: Int -> ByteString -> [ByteString]+bsChunksOf chunkSize bs+ | chunkSize < 1 = error $ "bsChunksOf: chunkSize < 1: " ++ show chunkSize+ | BS.length bs <= chunkSize = [bs]+ | otherwise =+ let (x, rest) = BS.splitAt chunkSize bs in x : bsChunksOf chunkSize rest+++-- | Compresses the incoming stream of ByteStrings with the lz4 frame format.+--+-- This function implements two optimisations to reduce unnecessary+-- allocations:+--+-- * Incoming ByteStrings are processed in blocks of 16 KB, allowing us+-- to use a single intermediate output buffer through the lifetime of+-- the conduit.+-- * The `bufferSize` of the output buffer can controlled by the caller+-- via the `bufferSize` argument, to reduce the number of small+-- `ByteString`s being `yield`ed (especially in the case that the+-- input data compresses very well, e.g. a stream of zeros).+--+-- Note that the given `bufferSize` is not a hard limit, it can only be+-- used to *increase* the amount of output buffer we're allowed to use:+-- The function will choose `max(bufferSize, minBufferSizeNeededByLz4)`+-- as the eventual output buffer size.+--+-- Setting `bufferSize = 0` is the legitimate way to set the output buffer+-- size to be the minimum required to compress 16 KB inputs and is still a+-- fast default.+compressWithOutBufferSize :: forall m . (MonadUnliftIO m, MonadResource m) => CSize -> ConduitT ByteString ByteString m ()+compressWithOutBufferSize bufferSize =+ withLz4CtxAndPrefsConduit $ \(ctx, prefs) -> do++ -- We split any incoming ByteString into chunks of this size, so that+ -- we can pass this size to `lz4fCompressBound` once and reuse a buffer+ -- of constant size for the compression.+ let bsInChunkSize = 16*1024++ compressBound <- liftIO $ lz4fCompressBound (#{const LZ4F_HEADER_SIZE_MAX} + bsInChunkSize) prefs+ let outBufferSize = max bufferSize compressBound++ outBuf <- liftIO $ mallocForeignPtrBytes (fromIntegral outBufferSize)+ let withOutBuf f = liftIO $ withForeignPtr outBuf f+ let yieldOutBuf outBufLen = do+ outBs <- withOutBuf $ \buf -> packCStringLen (buf, fromIntegral outBufLen)+ yield outBs++ headerSize <- withOutBuf $ \buf -> lz4fCompressBegin ctx prefs buf outBufferSize++ let writeFooterAndYield remainingCapacity = do+ let offset = fromIntegral $ outBufferSize - remainingCapacity+ footerWritten <- withOutBuf $ \buf -> lz4fCompressEnd ctx (buf `plusPtr` offset) remainingCapacity+ let outBufLen = outBufferSize - remainingCapacity + footerWritten+ yieldOutBuf outBufLen++ let loop remainingCapacity = do+ await >>= \case+ Nothing -> do+ -- Done, write footer.++ -- Passing srcSize==0 provides bound for LZ4F_compressEnd(),+ -- see docs of LZ4F_compressBound() for that.+ footerSize <- liftIO $ lz4fCompressBound 0 prefs++ if remainingCapacity >= footerSize+ then do+ writeFooterAndYield remainingCapacity+ else do+ -- Footer doesn't fit: Yield buffer, put footer into now-free buffer+ yieldOutBuf (outBufferSize - remainingCapacity)+ writeFooterAndYield outBufferSize++ Just bs -> do+ let bss = bsChunksOf (fromIntegral bsInChunkSize) bs+ newRemainingCapacity <- foldM (\cap subBs -> loopSingleBs cap subBs) remainingCapacity bss+ loop newRemainingCapacity++ loopSingleBs :: CSize -> ByteString -> ConduitM i ByteString m CSize+ loopSingleBs remainingCapacity bs+ | remainingCapacity < compressBound = do+ -- Not enough space in outBuf to guarantee that the next call+ -- to `lz4fCompressUpdate` will fit; so yield (a copy of) the+ -- current outBuf, then it's all free again.+ outBs <- withOutBuf $ \buf -> packCStringLen (buf, fromIntegral (outBufferSize - remainingCapacity))+ yield outBs+ loopSingleBs outBufferSize bs+ | otherwise = do+ written <- liftIO $ unsafeUseAsCStringLen bs $ \(bsPtr, bsLen) -> do+ let bsLenSize = fromIntegral bsLen++ -- See note [Single call to LZ4F_compressUpdate() can create multiple blocks]+ let offset = fromIntegral $ outBufferSize - remainingCapacity+ withOutBuf $ \buf -> lz4fCompressUpdate ctx (buf `plusPtr` offset) remainingCapacity bsPtr bsLenSize++ let newRemainingCapacity = remainingCapacity - written+ -- TODO assert newRemainingCapacity > 0+ let writtenInt = fromIntegral written++ if+ | written == 0 -> return newRemainingCapacity+ | writtenInt < BS.length bs -> loopSingleBs newRemainingCapacity (BS.drop writtenInt bs)+ | otherwise -> return newRemainingCapacity++ loop (outBufferSize - headerSize)+++++-- All notes that apply to `haskell_lz4_freeCompressionContext` apply+-- here as well.+C.verbatim [r|+void haskell_lz4_freeDecompressionContext(LZ4F_dctx** ctxPtr)+{+ LZ4F_dctx* ctx = *ctxPtr;+ if (ctx != NULL)+ {+ size_t err = LZ4F_freeDecompressionContext(ctx);+ if (LZ4F_isError(err))+ {+ fprintf(stderr, "LZ4F_freeDecompressionContext failed: %s\n", LZ4F_getErrorName(err));+ exit(1);+ }+ }+}+|]++foreign import ccall "&haskell_lz4_freeDecompressionContext" haskell_lz4_freeDecompressionContext :: FunPtr (Ptr (Ptr LZ4F_dctx) -> IO ())+++lz4fCreateDecompressionContext :: (HasCallStack) => IO Lz4FrameDecompressionContext+lz4fCreateDecompressionContext = do+ -- All notes that apply to `lz4fCreateCompressonContext` apply here+ -- as well.+ ctxForeignPtr :: ForeignPtr (Ptr LZ4F_dctx) <- mallocForeignPtr+ withForeignPtr ctxForeignPtr $ \ptr -> poke ptr nullPtr++ addForeignPtrFinalizer haskell_lz4_freeDecompressionContext ctxForeignPtr++ _ <- handleLz4Error [C.block| size_t {+ LZ4F_dctx** ctxPtr = $fptr-ptr:(LZ4F_dctx** ctxForeignPtr);+ LZ4F_errorCode_t err = LZ4F_createDecompressionContext(ctxPtr, LZ4F_VERSION);+ return err;+ } |]+ return (Lz4FrameDecompressionContext ctxForeignPtr)+++lz4fGetFrameInfo :: (HasCallStack) => Lz4FrameDecompressionContext -> Ptr FrameInfo -> Ptr CChar -> Ptr CSize -> IO CSize+lz4fGetFrameInfo (Lz4FrameDecompressionContext ctxForeignPtr) frameInfoPtr srcBuffer srcSizePtr = do++ decompressSizeHint <- handleLz4Error [C.block| size_t {+ LZ4F_dctx* ctxPtr = *$fptr-ptr:(LZ4F_dctx** ctxForeignPtr);+ LZ4F_errorCode_t err_or_decompressSizeHint = LZ4F_getFrameInfo(ctxPtr, $(LZ4F_frameInfo_t* frameInfoPtr), $(char* srcBuffer), $(size_t* srcSizePtr));+ return err_or_decompressSizeHint;+ } |]+ return decompressSizeHint+++lz4fDecompress :: (HasCallStack) => Lz4FrameDecompressionContext -> Ptr CChar -> Ptr CSize -> Ptr CChar -> Ptr CSize -> IO CSize+lz4fDecompress (Lz4FrameDecompressionContext ctxForeignPtr) dstBuffer dstSizePtr srcBuffer srcSizePtr = do+ -- TODO allow passing in dOptPtr instead of NULL.++ decompressSizeHint <- handleLz4Error [C.block| size_t {+ LZ4F_dctx* ctxPtr = *$fptr-ptr:(LZ4F_dctx** ctxForeignPtr);+ LZ4F_errorCode_t err_or_decompressSizeHint = LZ4F_decompress(ctxPtr, $(char* dstBuffer), $(size_t* dstSizePtr), $(char* srcBuffer), $(size_t* srcSizePtr), NULL);+ return err_or_decompressSizeHint;+ } |]+ return decompressSizeHint+++decompress :: (MonadUnliftIO m, MonadResource m) => ConduitT ByteString ByteString m ()+decompress = do+ ctx <- liftIO lz4fCreateDecompressionContext++ -- OK, now here it gets a bit ugly.+ -- The lz4frame library provides no function with which we can+ -- determine how large the header is.+ -- It depends on the "Content Size" bit in the "FLG" Byte (first+ -- Byte of the frame descriptor, just after the 4 Byte magic number).+ -- As a solution, we `await` the first 5 Bytes, look at the relevant+ -- bit in the FLG Byte and thus decide how many more bytes to await+ -- for the header.+ -- Is ugly because ideally we would rely only on the lz4frame API+ -- and not on the spec of the frame format, but we have no other+ -- choice in this case.++ first5Bytes <- CB.take 5+ when (BSL.length first5Bytes /= 5) $ do+ throwString $ "lz4 decompress error: not enough bytes for header; expected 5, got " ++ show (BSL.length first5Bytes)++ let byteFLG = BSL.index first5Bytes 4+ let contentSizeBit = testBit byteFLG 3++ let numRemainingHeaderBytes+ | contentSizeBit = 2 + 8+ | otherwise = 2++ remainingHeaderBytes <- CB.take numRemainingHeaderBytes++ let headerBs = BSL.toStrict $ BSL.concat [first5Bytes, remainingHeaderBytes]++ headerDecompressSizeHint <- liftIO $ alloca $ \frameInfoPtr -> do+ unsafeUseAsCStringLen headerBs $ \(headerBsPtr, headerBsLen) -> do+ with (fromIntegral headerBsLen :: CSize) $ \headerBsLenPtr -> do+ lz4fGetFrameInfo ctx frameInfoPtr headerBsPtr headerBsLenPtr++ let dstBufferSizeDefault :: CSize+ dstBufferSizeDefault = 16 * 1024++ bracketP+ (do+ dstBufferPtr <- malloc+ dstBufferSizePtr <- malloc+ poke dstBufferPtr =<< mallocArray (fromIntegral dstBufferSizeDefault)+ poke dstBufferSizePtr dstBufferSizeDefault+ return (dstBufferPtr, dstBufferSizePtr)+ )+ (\(dstBufferPtr, dstBufferSizePtr) -> do+ free =<< peek dstBufferPtr+ free dstBufferPtr+ free dstBufferSizePtr+ )+ $ \(dstBufferPtr, dstBufferSizePtr) -> do++ let ensureDstBufferSize :: CSize -> IO (Ptr CChar)+ ensureDstBufferSize size = do+ dstBufferSize <- peek dstBufferSizePtr+ when (size > dstBufferSize) $ do+ dstBuffer <- peek dstBufferPtr+ poke dstBufferPtr =<< reallocArray dstBuffer (fromIntegral size)+ poke dstBufferSizePtr size+ peek dstBufferPtr++ let loopSingleBs :: CSize -> ByteString -> _+ loopSingleBs decompressSizeHint bs = do+ (outBs, srcRead, newDecompressSizeHint) <- liftIO $+ unsafeUseAsCStringLen bs $ \(srcBuffer, srcSize) -> do+ let outBufSize = max decompressSizeHint dstBufferSizeDefault -- TODO check why decompressSizeHint is always 4++ -- Increase destination buffer size if necessary.+ dstBuffer <- ensureDstBufferSize outBufSize++ with outBufSize $ \dstSizePtr -> do+ with (fromIntegral srcSize :: CSize) $ \srcSizePtr -> do+ newDecompressSizeHint <-+ lz4fDecompress ctx dstBuffer dstSizePtr srcBuffer srcSizePtr+ srcRead <- peek srcSizePtr+ dstWritten <- peek dstSizePtr+ outBs <- packCStringLen (dstBuffer, fromIntegral dstWritten)+ return (outBs, srcRead, newDecompressSizeHint)++ yield outBs++ let srcReadInt = fromIntegral srcRead+ if+ | srcReadInt < BS.length bs -> loopSingleBs newDecompressSizeHint (BS.drop srcReadInt bs)+ | srcReadInt == BS.length bs -> return newDecompressSizeHint+ | otherwise -> error $ "lz4 decompress: assertion failed: srcRead < BS.length bs: " ++ show (srcRead, BS.length bs)++ let loop decompressSizeHint =+ await >>= \case+ Nothing -> throwString $ "lz4 decompress error: stream ended before EndMark"+ Just bs -> do+ newDecompressSizeHint <- loopSingleBs decompressSizeHint bs++ -- When a frame is fully decoded, LZ4F_decompress returns 0 (no more data expected),+ -- see https://github.com/lz4/lz4/blob/7cf0bb97b2a988cb17435780d19e145147dd9f70/lib/lz4frame.h#L324+ when (newDecompressSizeHint /= 0) $ loop newDecompressSizeHint++ loop headerDecompressSizeHint++ -- Force resource release here to guarantee memory constantness+ -- of the conduit (and not rely on GC to do it "at some point in the future").+ liftIO $ finalizeForeignPtr (unLz4FrameDecompressionContext ctx)
+ test/Main.hs view
@@ -0,0 +1,81 @@+{-# LANGUAGE OverloadedStrings #-}++module Main (main) where++import Codec.Compression.LZ4.Conduit (compress, decompress, bsChunksOf)+import Control.Monad.IO.Unlift (MonadUnliftIO)+import Control.Monad.Trans.Resource (ResourceT, runResourceT)+import Data.ByteString (ByteString)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as BSL+import qualified Data.ByteString.Lazy.Char8 as BSL8+import Data.Conduit+import qualified Data.Conduit.List as CL+import qualified Data.Conduit.Process as CP+import Data.List (intersperse)+import Test.Hspec+import Test.Hspec.QuickCheck (modifyMaxSize)+import qualified Test.QuickCheck as QC+import qualified Test.QuickCheck.Monadic as QCM++++runCompressToLZ4 :: (MonadUnliftIO m) => ConduitT () ByteString (ResourceT m) () -> m ByteString+runCompressToLZ4 source = runResourceT $ do++ (_, result, _) <- CP.sourceCmdWithStreams "lz4 -d" (source .| compress) CL.consume CL.consume+ return $ BS.concat result++runLZ4ToDecompress :: (MonadUnliftIO m) => ConduitT () ByteString (ResourceT m) () -> m ByteString+runLZ4ToDecompress source = runResourceT $ do+ (_, result, _) <- CP.sourceCmdWithStreams "lz4 -c" source (decompress .| CL.consume) CL.consume+ return $ BS.concat result++main :: IO ()+main = do+ let prepare :: [BSL.ByteString] -> [ByteString]+ prepare strings = BSL.toChunks $ BSL.concat $ intersperse " " $ ["BEGIN"] ++ strings ++ ["END"]++ hspec $ do+ describe "bsChunksOf" $ do+ it "chunks up a string" $ do+ bsChunksOf 3 "abc123def4567" `shouldBe` ["abc", "123", "def", "456", "7"]++ describe "Compression" $ do+ it "compresses simple string" $ do+ let string = "hellohellohellohello"+ actual <- runCompressToLZ4 (yield string)+ actual `shouldBe` string++ it "compresses 100000 integers" $ do+ let strings = prepare $ map (BSL8.pack . show) [1..100000 :: Int]+ actual <- runCompressToLZ4 (CL.sourceList strings)+ actual `shouldBe` (BS.concat strings)++ it "compresses 100000 strings" $ do+ let strings = prepare $ replicate 100000 "hello"+ actual <- runCompressToLZ4 (CL.sourceList strings)+ actual `shouldBe` (BS.concat strings)++ describe "Decompression" $ do+ it "decompresses simple string" $ do+ let string = "hellohellohellohello"+ actual <- runLZ4ToDecompress (yield string)+ actual `shouldBe` string++ it "decompresses 100000 integers" $ do+ let strings = prepare $ map (BSL8.pack . show) [1..100000 :: Int]+ actual <- runLZ4ToDecompress (CL.sourceList strings)+ actual `shouldBe` (BS.concat strings)++ it "decompresses 100000 strings" $ do+ let strings = prepare $ replicate 100000 "hello"+ actual <- runLZ4ToDecompress (CL.sourceList strings)+ actual `shouldBe` (BS.concat strings)++ describe "Identity" $ do+ modifyMaxSize (const 10000) $ it "compress and decompress arbitrary strings"$+ QC.property $ \string -> QCM.monadicIO $ do+ let bs = BSL.toChunks $ BSL8.pack string+ actual <- QCM.run (runConduitRes $ CL.sourceList bs .| compress .| decompress .| CL.consume)+ QCM.assert(BS.concat bs == BS.concat actual)