diff --git a/public/common/tracy_lz4.cpp b/public/common/tracy_lz4.cpp index 4c5c1a9f..035e2f90 100644 --- a/public/common/tracy_lz4.cpp +++ b/public/common/tracy_lz4.cpp @@ -1,6 +1,6 @@ /* LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2023, Yann Collet. + Copyright (c) Yann Collet. All rights reserved. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -45,18 +45,6 @@ # define LZ4_HEAPMODE 0 #endif -/* - * LZ4_ACCELERATION_DEFAULT : - * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 - */ -#define LZ4_ACCELERATION_DEFAULT 1 -/* - * LZ4_ACCELERATION_MAX : - * Any "acceleration" value higher than this threshold - * get treated as LZ4_ACCELERATION_MAX instead (fix #876) - */ -#define LZ4_ACCELERATION_MAX 65537 - /*-************************************ * CPU Feature Detection @@ -77,7 +65,8 @@ #ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ # if defined(__GNUC__) && \ ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ - || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) \ + || (defined(__riscv) && defined(__riscv_zicclsm)) ) # define LZ4_FORCE_MEMORY_ACCESS 2 # elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) || defined(_MSC_VER) # define LZ4_FORCE_MEMORY_ACCESS 1 @@ -145,6 +134,17 @@ # endif /* _MSC_VER */ #endif /* LZ4_FORCE_INLINE */ +/* LZ4_FORCE_NOINLINE : keeps a rarely used function out of its callers' hot loops */ +#ifndef LZ4_FORCE_NOINLINE +# if defined(_MSC_VER) +# define LZ4_FORCE_NOINLINE static __declspec(noinline) +# elif defined(__GNUC__) || defined(__clang__) +# define LZ4_FORCE_NOINLINE static __attribute__((noinline)) +# else +# define LZ4_FORCE_NOINLINE static +# endif +#endif + /* LZ4_FORCE_O2 and LZ4_FORCE_INLINE * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, * together with a simple 8-byte copy loop as a fall-back path. @@ -213,7 +213,7 @@ #elif defined(LZ4_USER_MEMORY_FUNCTIONS) /* memory management functions can be customized by user project. * Below functions must exist somewhere in the Project - * and be available at link time */ + * and be available at link time. */ void* LZ4_malloc(size_t s); void* LZ4_calloc(size_t n, size_t s); void LZ4_free(void* p); @@ -301,12 +301,12 @@ static int LZ4_isAligned(const void* ptr, size_t alignment) #include #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef uintptr_t uptrval; + typedef unsigned char BYTE; /*uint8_t not necessarily blessed to alias arbitrary type*/ + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef uintptr_t uptrval; #else # if UINT_MAX != 4294967295UL # error "LZ4 code (when not C++ or C99) assumes that sizeof(int) == 4" @@ -480,25 +480,23 @@ static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; #ifndef LZ4_FAST_DEC_LOOP # if defined __i386__ || defined _M_IX86 || defined __x86_64__ || defined _M_X64 # define LZ4_FAST_DEC_LOOP 1 -# elif defined(__aarch64__) && defined(__APPLE__) -# define LZ4_FAST_DEC_LOOP 1 -# elif defined(__aarch64__) && !defined(__clang__) - /* On non-Apple aarch64, we disable this optimization for clang because - * on certain mobile chipsets, performance is reduced with clang. For - * more information refer to https://github.com/lz4/lz4/pull/707 */ +# elif defined(__aarch64__) # define LZ4_FAST_DEC_LOOP 1 +# elif defined(__riscv) && (__riscv_xlen == 64) && defined(__riscv_misaligned_fast) +# define LZ4_FAST_DEC_LOOP 1 /* otherwise, unaligned memcpy() become byte copies, defeating the fast loop */ # else # define LZ4_FAST_DEC_LOOP 0 # endif #endif -#if LZ4_FAST_DEC_LOOP - +/* LZ4_memcpy_using_offset_base() : + * copy a match of length >= MINMATCH (srcPtr + offset == dstPtr, overlap allowed), + * can overwrite up to 12 bytes beyond dstEnd */ LZ4_FORCE_INLINE void LZ4_memcpy_using_offset_base(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) { assert(srcPtr + offset == dstPtr); - if (offset < 8) { + if (unlikely(offset < 8)) { LZ4_write32(dstPtr, 0); /* silence an msan warning when offset==0 */ dstPtr[0] = srcPtr[0]; dstPtr[1] = srcPtr[1]; @@ -507,16 +505,44 @@ LZ4_memcpy_using_offset_base(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, con srcPtr += inc32table[offset]; LZ4_memcpy(dstPtr+4, srcPtr, 4); srcPtr -= dec64table[offset]; - dstPtr += 8; } else { LZ4_memcpy(dstPtr, srcPtr, 8); - dstPtr += 8; srcPtr += 8; } + dstPtr += 8; - LZ4_wildCopy8(dstPtr, srcPtr, dstEnd); + /* srcPtr is now at least 8 bytes behind dstPtr */ + LZ4_memcpy(dstPtr, srcPtr, 8); + if (dstEnd > dstPtr + 8) { LZ4_wildCopy8(dstPtr + 8, srcPtr + 8, dstEnd); } } + +#if LZ4_FAST_DEC_LOOP + +/* LZ4_wildCopy64 is faster on aarch64 and x86-64, but not on 32-bit x86. + * On wasm, it depends on the host cpu (faster on Apple silicon, slower on x86-64) : + * it's not enabled by default, but can be, with LZ4_FAST_DEC_LOOP, from the build command */ +#ifndef LZ4_WILDCOPY64 +# if defined(__aarch64__) || defined(__x86_64__) || defined(_M_X64) +# define LZ4_WILDCOPY64 1 +# else +# define LZ4_WILDCOPY64 0 +# endif +#endif + +#if LZ4_WILDCOPY64 +/* customized variant of memcpy, which can overwrite up to 64 bytes beyond dstEnd */ +LZ4_FORCE_INLINE void +LZ4_wildCopy64(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_memcpy(d,s,64); d+=64; s+=64; } while (d= 16. */ @@ -767,15 +793,34 @@ int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, const void* dictStart, size_t dictSize); + +/*-**************************************** +* Obsolete functions, removed from lz4.h (v1.11.0) +* still exported, for programs linked to previous versions +*******************************************/ + +LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); +LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); +LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); +LZ4LIB_API void* LZ4_create (char* inputBuffer); +LZ4LIB_API int LZ4_sizeofStreamState(void); +LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4LIB_API char* LZ4_slideInputBuffer (void* state); +LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + /*-****************************** * Compression functions ********************************/ LZ4_FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { - if (tableType == byU16) - return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); - else - return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + return ((sequence * 2654435761U) >> ((MINMATCH*8)-hashLog)); } LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType) @@ -825,7 +870,7 @@ LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableT } } -/* LZ4_putPosition*() : only used in byPtr mode */ +/* LZ4_putPositionOnHash() : only used in byPtr mode */ LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType) { @@ -834,10 +879,16 @@ LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, hashTable[h] = p; } -LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType) +/* LZ4_putPosition() : + * register position p : by pointer (byPtr), or by index relative to base (byU32, byU16) */ +LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* base) { U32 const h = LZ4_hashPosition(p, tableType); - LZ4_putPositionOnHash(p, h, tableBase, tableType); + if (tableType == byPtr) { + LZ4_putPositionOnHash(p, h, tableBase, byPtr); + } else { + LZ4_putIndexOnHash((U32)(p - base), h, tableBase, tableType); + } } /* LZ4_getIndexOnHash() : @@ -868,13 +919,9 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } } -LZ4_FORCE_INLINE const BYTE* -LZ4_getPosition(const BYTE* p, - const void* tableBase, tableType_t tableType) -{ - U32 const h = LZ4_hashPosition(p, tableType); - return LZ4_getPositionOnHash(h, tableBase, tableType); -} +/* beyond this input size (1 KB by default), + * clearing the table is faster than rejecting its stale entries */ +#define LZ4_CLEAR_TABLE_THRESHOLD (LZ4_HASHTABLESIZE / 16) LZ4_FORCE_INLINE void LZ4_prepareTable(LZ4_stream_t_internal* const cctx, @@ -890,9 +937,9 @@ LZ4_prepareTable(LZ4_stream_t_internal* const cctx, || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) || ((tableType == byU32) && cctx->currentOffset > 1 GB) || tableType == byPtr - || inputSize >= 4 KB) + || inputSize >= LZ4_CLEAR_TABLE_THRESHOLD) { - DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", (void*)cctx); MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); cctx->currentOffset = 0; cctx->tableType = (U32)clearedTable; @@ -917,6 +964,21 @@ LZ4_prepareTable(LZ4_stream_t_internal* const cctx, cctx->dictSize = 0; } +/* LZ4_match8() : the 8 bytes at p and q are equal */ +LZ4_FORCE_INLINE int LZ4_match8(const void* p, const void* q) +{ + if (sizeof(reg_t) == 8) return LZ4_read_ARCH(p) == LZ4_read_ARCH(q); + return (LZ4_read32(p) == LZ4_read32(q)) + && (LZ4_read32((const BYTE*)p + 4) == LZ4_read32((const BYTE*)q + 4)); +} + +/* state of an extDict compression which resumes in prefix mode (see LZ4_compress_generic()) */ +typedef struct { + const BYTE* ip; /* end of a match : resumes at the test of the next position */ + BYTE* op; + U32 startIndex; +} LZ4_resume_t; + /** LZ4_compress_generic_validated() : * inlined, to ensure branches are decided at compilation time. * The following conditions are presumed already validated: @@ -934,12 +996,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( const tableType_t tableType, const dict_directive dictDirective, const dictIssue_directive dictIssue, - const int acceleration) + const int acceleration, + LZ4_resume_t* const suspend, /* usingExtDict : stops when the dictionary is out of reach, saving its state */ + const LZ4_resume_t* const resume) /* withPrefix64k : resumes such a compression */ { int result; - const BYTE* ip = (const BYTE*)source; + const BYTE* ip = resume ? resume->ip : (const BYTE*)source; - U32 const startIndex = cctx->currentOffset; + U32 const startIndex = resume ? resume->startIndex : cctx->currentOffset; const BYTE* base = (const BYTE*)source - startIndex; const BYTE* lowLimit; @@ -954,10 +1018,11 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ const BYTE* const dictEnd = dictionary ? dictionary + dictSize : dictionary; - const BYTE* anchor = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; - const BYTE* const matchlimit = iend - LASTLITERALS; + const BYTE* anchor = resume ? resume->ip : (const BYTE*)source; + const BYTE* const iend = (const BYTE*)source + inputSize; + const BYTE* mflimitPlusOne; /* set after the size check : would precede a small input */ + const BYTE* matchlimit; + const BYTE* seqLimit; /* suspend : mflimitPlusOne, or earlier, where the dictionary gets out of reach */ /* the dictCtx currentOffset is indexed on the start of the dictionary, * while a dictionary in the current context precedes the currentOffset */ @@ -966,11 +1031,12 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( dictionary + dictSize - dictCtx->currentOffset : dictionary + dictSize - startIndex; - BYTE* op = (BYTE*) dest; - BYTE* const olimit = op + maxOutputSize; + BYTE* op = resume ? resume->op : (BYTE*)dest; + BYTE* const olimit = (BYTE*)dest + maxOutputSize; U32 offset = 0; U32 forwardH; + const BYTE* filledIp = NULL; /* fillOutput only ; set by catch up (resume : never with fillOutput) */ DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", inputSize, tableType); assert(ip != NULL); @@ -981,36 +1047,40 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ assert(acceleration >= 1); - lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); + /* resume : candidates are within source (see LZ4_compress_generic()) */ + lowLimit = (const BYTE*)source - ((dictDirective == withPrefix64k) && !resume ? dictSize : 0); - /* Update context state */ - if (dictDirective == usingDictCtx) { - /* Subsequent linked blocks can't use the dictionary. */ - /* Instead, they use the block we just compressed. */ - cctx->dictCtx = NULL; - cctx->dictSize = (U32)inputSize; - } else { - cctx->dictSize += (U32)inputSize; + if (!resume) { /* resume : context state already updated */ + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; + } + cctx->currentOffset += (U32)inputSize; + cctx->tableType = (U32)tableType; + + if (inputSizecurrentOffset += (U32)inputSize; - cctx->tableType = (U32)tableType; + mflimitPlusOne = iend - MFLIMIT + 1; + matchlimit = iend - LASTLITERALS; + seqLimit = mflimitPlusOne; + if ((suspend != NULL) && (inputSize > LZ4_DISTANCE_MAX + MFLIMIT - 1)) /* source + LZ4_DISTANCE_MAX < mflimitPlusOne */ + seqLimit = (const BYTE*)source + LZ4_DISTANCE_MAX; - if (inputSizehashTable, byPtr); - } else { - LZ4_putIndexOnHash(startIndex, h, cctx->hashTable, tableType); - } } + LZ4_putPosition(ip, cctx->hashTable, tableType, base); ip++; forwardH = LZ4_hashPosition(ip, tableType); /* Main Loop */ for ( ; ; ) { const BYTE* match; BYTE* token; - const BYTE* filledIp; /* Find a match */ if (tableType == byPtr) { @@ -1064,16 +1134,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( lowLimit = (const BYTE*)source; } } else if (dictDirective == usingExtDict) { - if (matchIndex < startIndex) { - DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); - assert(startIndex - matchIndex >= MINMATCH); - assert(dictBase); - match = dictBase + matchIndex; - lowLimit = dictionary; - } else { - match = base + matchIndex; - lowLimit = (const BYTE*)source; - } + /* select without branch : dictionary and prefix candidates alternate unpredictably. + * relative to dictEnd or source, within their buffers (base can be NULL) */ + ptrdiff_t const relIdx = (ptrdiff_t)matchIndex - (ptrdiff_t)startIndex; /* < 0 : within dictionary */ + match = ((relIdx < 0) ? dictEnd : (const BYTE*)source) + relIdx; } else { /* single continuous memory segment */ match = base + matchIndex; } @@ -1083,14 +1147,25 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ assert(matchIndex < current); - if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + /* noDict : the candidate is within input, so it is compared before checking its distance : + * this order is faster (better branch prediction) */ + assert((dictDirective != noDict) || (match >= (const BYTE*)source)); + if ( (dictDirective != noDict) + && ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) && (matchIndex+LZ4_DISTANCE_MAX < current)) { continue; } /* too far */ - assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ if (LZ4_read32(match) == LZ4_read32(ip)) { + if ( (dictDirective == noDict) + && ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + && (matchIndex+LZ4_DISTANCE_MAX < current)) { + continue; + } /* too far */ + assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ if (maybe_extMem) offset = current - matchIndex; + if (dictDirective == usingExtDict) /* lowLimit : only needed once a match is found */ + lowLimit = (matchIndex < startIndex) ? dictionary : (const BYTE*)source; break; /* match found */ } @@ -1226,22 +1301,29 @@ _next_match: anchor = ip; /* Test end of chunk */ - if (ip >= mflimitPlusOne) break; + if (ip >= seqLimit) { + if ((suspend == NULL) || (ip >= mflimitPlusOne)) break; + /* extDict : from ip, all candidates (and catch up) are beyond reach of the dictionary : + * stop, to resume in prefix mode, which is faster */ + suspend->ip = ip; + suspend->op = op; + suspend->startIndex = startIndex; + return 0; + } - /* Fill table */ - { U32 const h = LZ4_hashPosition(ip-2, tableType); - if (tableType == byPtr) { - LZ4_putPositionOnHash(ip-2, h, cctx->hashTable, byPtr); - } else { - U32 const idx = (U32)((ip-2) - base); - LZ4_putIndexOnHash(idx, h, cctx->hashTable, tableType); - } } - - /* Test next position */ +_resume: + /* Fill table, and test next position. + * First, the position after the previous occurrence of ip-1, if it matches 8+ bytes : + * it breaks chains of short matches, which the table alone can produce on periodic data. */ if (tableType == byPtr) { - match = LZ4_getPosition(ip, cctx->hashTable, tableType); - LZ4_putPosition(ip, cctx->hashTable, tableType); + U32 const hPrev = LZ4_hashPosition(ip-1, tableType); + const BYTE* const prev = LZ4_getPositionOnHash(hPrev, cctx->hashTable, tableType); + U32 const h = LZ4_hashPosition(ip, tableType); + LZ4_putPositionOnHash(ip-1, hPrev, cctx->hashTable, tableType); + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType); + if ((prev+1+LZ4_DISTANCE_MAX >= ip) && LZ4_match8(prev+1, ip)) match = prev+1; + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType); if ( (match+LZ4_DISTANCE_MAX >= ip) && (LZ4_read32(match) == LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } @@ -1250,31 +1332,41 @@ _next_match: U32 const h = LZ4_hashPosition(ip, tableType); U32 const current = (U32)(ip-base); - U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + U32 const hPrev = LZ4_hashPosition(ip-1, tableType); + U32 prevIndex = LZ4_getIndexOnHash(hPrev, cctx->hashTable, tableType); + U32 matchIndex; + LZ4_putIndexOnHash(current-1, hPrev, cctx->hashTable, tableType); + if ((dictDirective == usingDictCtx) && (prevIndex < startIndex)) { + /* no occurrence in the current context : try the dictionary */ + prevIndex = LZ4_getIndexOnHash(hPrev, dictCtx->hashTable, byU32) + dictDelta; + } + { /* used table (small block, see LZ4_prepareTable()) : a stale entry yields the candidate of an empty slot + * of a cleared table (0), the position after the first one : the output doesn't depend on the table's history */ + int const usedTable = (dictDirective == noDict) && (dictIssue == dictSmall); + U32 const ctxIndex = ((usedTable && (prevIndex < startIndex)) ? startIndex : prevIndex) + 1; + int const inDict = (dictDirective == usingExtDict || dictDirective == usingDictCtx) && (ctxIndex < startIndex); + if ( ((dictIssue==dictSmall) ? (ctxIndex >= prefixIdxLimit) : 1) + && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (ctxIndex+LZ4_DISTANCE_MAX >= current)) + && (!inDict || (ctxIndex + 8 <= startIndex)) /* the 8 compared bytes are within the dictionary */ + && LZ4_match8(inDict ? dictBase + (ctxIndex - dictDelta) : base + ctxIndex, ip) ) { + matchIndex = ctxIndex; + } else { + matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + if ((dictDirective == usingDictCtx) && (matchIndex < startIndex)) + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32) + dictDelta; + } + } assert(matchIndex < current); - if (dictDirective == usingDictCtx) { - if (matchIndex < startIndex) { - /* there was no match, try the dictionary */ - assert(tableType == byU32); - matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); - match = dictBase + matchIndex; - lowLimit = dictionary; /* required for match length counter */ - matchIndex += dictDelta; - } else { - match = base + matchIndex; - lowLimit = (const BYTE*)source; /* required for match length counter */ - } - } else if (dictDirective==usingExtDict) { - if (matchIndex < startIndex) { - assert(dictBase); - match = dictBase + matchIndex; - lowLimit = dictionary; /* required for match length counter */ - } else { - match = base + matchIndex; - lowLimit = (const BYTE*)source; /* required for match length counter */ - } - } else { /* single memory segment */ + if (dictDirective == usingExtDict) { /* select without branch */ + ptrdiff_t const relIdx = (ptrdiff_t)matchIndex - (ptrdiff_t)startIndex; /* < 0 : within dictionary */ + match = ((relIdx < 0) ? dictEnd : (const BYTE*)source) + relIdx; + } else if ((dictDirective == usingDictCtx) && (matchIndex < startIndex)) { + match = dictBase + (matchIndex - dictDelta); + lowLimit = dictionary; /* required for match length counter */ + } else { match = base + matchIndex; + if (dictDirective == usingDictCtx) + lowLimit = (const BYTE*)source; /* required for match length counter */ } LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); assert(matchIndex < current); @@ -1284,6 +1376,8 @@ _next_match: token=op++; *token=0; if (maybe_extMem) offset = current - matchIndex; + if (dictDirective == usingExtDict) + lowLimit = (matchIndex < startIndex) ? dictionary : (const BYTE*)source; /* required for match length counter */ DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); goto _next_match; @@ -1333,6 +1427,16 @@ _last_literals: return result; } +/* second part of an extDict compression, in prefix mode (see LZ4_compress_generic()). + * not inlined : a single instance serves all invocations */ +LZ4_FORCE_NOINLINE int LZ4_compress_resumeWithPrefix(LZ4_stream_t_internal* cctx, + const char* src, char* dst, int srcSize, int dstCapacity, int acceleration, + const LZ4_resume_t* state) +{ + return LZ4_compress_generic_validated(cctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, + byU32, withPrefix64k, noDictIssue, acceleration, NULL, state); +} + /** LZ4_compress_generic() : * inlined, to ensure branches are decided at compilation time; * takes care of src == (NULL, 0) @@ -1368,36 +1472,78 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( } assert(src != NULL); + if ((dictDirective == usingExtDict) && (dictIssue == noDictIssue)) { + /* beyond LZ4_DISTANCE_MAX, the dictionary is out of reach : the rest of the input is compressed in prefix mode */ + LZ4_resume_t state; + int result; + assert(tableType == byU32); + assert(outputDirective != fillOutput); + state.ip = NULL; + result = LZ4_compress_generic_validated(cctx, src, dst, srcSize, inputConsumed, dstCapacity, outputDirective, + tableType, usingExtDict, noDictIssue, acceleration, &state, NULL); + if (state.ip == NULL) return result; /* completed (or failed) in extDict mode */ + /* notLimited : dst is presumed large enough for LZ4_compressBound() */ + return LZ4_compress_resumeWithPrefix(cctx, src, dst, srcSize, + (outputDirective == notLimited) ? LZ4_COMPRESSBOUND(srcSize) : dstCapacity, + acceleration, &state); + } + return LZ4_compress_generic_validated(cctx, src, dst, srcSize, inputConsumed, /* only written into if outputDirective == fillOutput */ dstCapacity, outputDirective, - tableType, dictDirective, dictIssue, acceleration); + tableType, dictDirective, dictIssue, acceleration, NULL, NULL); } -int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +/* values < 1 select the default, values > LZ4_ACCELERATION_MAX are capped */ +static int LZ4_clampAcceleration(int acceleration) { - LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; - assert(ctx != NULL); - if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; - if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; - if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) { - return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + if (acceleration < 1) return LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) return LZ4_ACCELERATION_MAX; + return acceleration; +} + +/* table type for inputs too large for byU16. + * byPtr (32-bit only) requires src > LZ4_DISTANCE_MAX, + * so that empty (NULL) entries are always beyond the maximum distance */ +static tableType_t LZ4_largeInputTableType(const char* src) +{ + (void)src; /* unused on 64-bit systems */ + return ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; +} + +/* outputDirective must be a compile-time constant */ +LZ4_FORCE_INLINE int LZ4_compress_fastReset_generic( + LZ4_stream_t_internal* const ctx, + const char* const src, + char* const dst, + const int srcSize, + const int dstCapacity, + const limitedOutput_directive outputDirective, + const int acceleration) +{ + if (srcSize < LZ4_64Klimit) { + LZ4_prepareTable(ctx, srcSize, byU16); + if (ctx->currentOffset) { /* table re-used : stale entries must be rejected */ + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, outputDirective, byU16, noDict, dictSmall, acceleration); } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; - return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, outputDirective, byU16, noDict, noDictIssue, acceleration); } } else { - if (inputSize < LZ4_64Klimit) { - return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; - return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); - } + const tableType_t tableType = LZ4_largeInputTableType(src); + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, outputDirective, tableType, noDict, noDictIssue, acceleration); } } +int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + /* a freshly initialized state is valid for _fastReset() */ + LZ4_stream_t* const s = LZ4_initStream(state, sizeof(LZ4_stream_t)); + assert(s != NULL); + return LZ4_compress_fast_extState_fastReset(s, source, dest, inputSize, maxOutputSize, acceleration); +} + /** * LZ4_compress_fast_extState_fastReset() : * A variant of LZ4_compress_fast_extState(). @@ -1410,38 +1556,13 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { LZ4_stream_t_internal* const ctx = &((LZ4_stream_t*)state)->internal_donotuse; - if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; - if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + acceleration = LZ4_clampAcceleration(acceleration); assert(ctx != NULL); if (dstCapacity >= LZ4_compressBound(srcSize)) { - if (srcSize < LZ4_64Klimit) { - const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, srcSize, tableType); - if (ctx->currentOffset) { - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); - } else { - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); - } - } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; - LZ4_prepareTable(ctx, srcSize, tableType); - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); - } + return LZ4_compress_fastReset_generic(ctx, src, dst, srcSize, 0, notLimited, acceleration); } else { - if (srcSize < LZ4_64Klimit) { - const tableType_t tableType = byU16; - LZ4_prepareTable(ctx, srcSize, tableType); - if (ctx->currentOffset) { - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); - } else { - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); - } - } else { - const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; - LZ4_prepareTable(ctx, srcSize, tableType); - return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); - } + return LZ4_compress_fastReset_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, acceleration); } } @@ -1478,15 +1599,15 @@ static int LZ4_compress_destSize_extState_internal(LZ4_stream_t* state, const ch { void* const s = LZ4_initStream(state, sizeof (*state)); assert(s != NULL); (void)s; + acceleration = LZ4_clampAcceleration(acceleration); if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ - return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, acceleration); + return LZ4_compress_fast_extState_fastReset(state, src, dst, *srcSizePtr, targetDstSize, acceleration); } else { if (*srcSizePtr < LZ4_64Klimit) { return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, acceleration); } else { - tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; - return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, LZ4_largeInputTableType(src), noDict, noDictIssue, acceleration); } } } @@ -1501,6 +1622,7 @@ int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { + int result; #if (LZ4_HEAPMODE) LZ4_stream_t* const ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ if (ctx == NULL) return 0; @@ -1508,8 +1630,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe LZ4_stream_t ctxBody; LZ4_stream_t* const ctx = &ctxBody; #endif - - int result = LZ4_compress_destSize_extState_internal(ctx, src, dst, srcSizePtr, targetDstSize, 1); + result = LZ4_compress_destSize_extState_internal(ctx, src, dst, srcSizePtr, targetDstSize, 1); #if (LZ4_HEAPMODE) FREEMEM(ctx); @@ -1528,7 +1649,7 @@ LZ4_stream_t* LZ4_createStream(void) { LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); LZ4_STATIC_ASSERT(sizeof(LZ4_stream_t) >= sizeof(LZ4_stream_t_internal)); - DEBUGLOG(4, "LZ4_createStream %p", lz4s); + DEBUGLOG(4, "LZ4_createStream %p", (void*)lz4s); if (lz4s == NULL) return NULL; LZ4_initStream(lz4s, sizeof(*lz4s)); return lz4s; @@ -1559,7 +1680,7 @@ LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) * prefer initStream() which is more general */ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { - DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); + DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", (void*)LZ4_stream); MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); } @@ -1571,7 +1692,7 @@ void LZ4_resetStream_fast(LZ4_stream_t* ctx) { int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { if (!LZ4_stream) return 0; /* support free on NULL */ - DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream); + DEBUGLOG(5, "LZ4_freeStream %p", (void*)LZ4_stream); FREEMEM(LZ4_stream); return (0); } @@ -1580,7 +1701,7 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) typedef enum { _ld_fast, _ld_slow } LoadDict_mode_e; #define HASH_UNIT sizeof(reg_t) -int LZ4_loadDict_internal(LZ4_stream_t* LZ4_dict, +static int LZ4_loadDict_internal(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize, LoadDict_mode_e _ld) { @@ -1590,7 +1711,7 @@ int LZ4_loadDict_internal(LZ4_stream_t* LZ4_dict, const BYTE* const dictEnd = p + dictSize; U32 idx32; - DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, (void*)dictionary, (void*)LZ4_dict); /* It's necessary to reset the context, * and not just continue it with prepareTable() @@ -1605,6 +1726,9 @@ int LZ4_loadDict_internal(LZ4_stream_t* LZ4_dict, * in LZ4_compress_fast_continue() where it uses noDictIssue even when the * dictionary isn't a full 64k. */ dict->currentOffset += 64 KB; + /* no longer a cleared table : LZ4_prepareTable() must check it, + * even when the dictionary is too small to be used */ + dict->tableType = (U32)tableType; if (dictSize < (int)HASH_UNIT) { return 0; @@ -1613,7 +1737,6 @@ int LZ4_loadDict_internal(LZ4_stream_t* LZ4_dict, if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); - dict->tableType = (U32)tableType; idx32 = dict->currentOffset - dict->dictSize; while (p <= dictEnd-HASH_UNIT) { @@ -1657,7 +1780,7 @@ void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dict &(dictionaryStream->internal_donotuse); DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", - workingStream, dictionaryStream, + (void*)workingStream, (void*)dictionaryStream, dictCtx != NULL ? dictCtx->dictSize : 0); if (dictCtx != NULL) { @@ -1668,6 +1791,8 @@ void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dict */ if (workingStream->internal_donotuse.currentOffset == 0) { workingStream->internal_donotuse.currentOffset = 64 KB; + /* no longer a cleared table : LZ4_prepareTable() must check it */ + workingStream->internal_donotuse.tableType = (U32)byU32; } /* Don't actually attach an empty dictionary. @@ -1700,6 +1825,19 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) } +/* The history covers neither the full window (64 KB) nor all positions since the start of indexes : + * the table may contain stale entries, from before the history, which must be rejected (dictSmall). */ +static int LZ4_isDictSmall(const LZ4_stream_t_internal* streamPtr) +{ + return (streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset); +} + +/* from this input size (2 KB by default, at most 4 KB), + * copying the tables of an attached dictionary is faster than looking into both tables. + * The copy cost follows the table size. */ +#define LZ4_DICTCTX_COPY_THRESHOLD ((LZ4_HASHTABLESIZE / 8 < 4 KB) ? (LZ4_HASHTABLESIZE / 8) : 4 KB) + + int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, @@ -1711,9 +1849,12 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", inputSize, streamPtr->dictSize); + /* new stream (no history) : clear the table beyond LZ4_CLEAR_TABLE_THRESHOLD */ + if ((streamPtr->dictSize == 0) && (streamPtr->dictCtx == NULL) && (inputSize >= LZ4_CLEAR_TABLE_THRESHOLD)) + LZ4_prepareTable(streamPtr, inputSize, byU32); + LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */ - if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; - if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + acceleration = LZ4_clampAcceleration(acceleration); /* invalidate tiny dictionaries */ if ( (streamPtr->dictSize < 4) /* tiny dictionary : not enough for a hash */ @@ -1721,7 +1862,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, && (inputSize > 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */ && (streamPtr->dictCtx == NULL) /* usingDictCtx */ ) { - DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); + DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, (void*)streamPtr->dictionary); /* remove dictionary existence from history, to employ faster prefix mode */ streamPtr->dictSize = 0; streamPtr->dictionary = (const BYTE*)source; @@ -1730,8 +1871,9 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, /* Check overlapping input/dictionary space */ { const char* const sourceEnd = source + inputSize; - if ((sourceEnd > (const char*)streamPtr->dictionary) && (sourceEnd < dictEnd)) { - streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if ((sourceEnd > (const char*)streamPtr->dictionary) && (source < dictEnd)) { + /* only the part of the dictionary beyond input remains */ + streamPtr->dictSize = (sourceEnd < dictEnd) ? (U32)(dictEnd - sourceEnd) : 0; if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; streamPtr->dictionary = (const BYTE*)dictEnd - streamPtr->dictSize; @@ -1740,7 +1882,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, /* prefix mode : source data follows dictionary */ if (dictEnd == source) { - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + if (LZ4_isDictSmall(streamPtr)) return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); else return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); @@ -1755,7 +1897,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe * to use noDictIssue even when the dict isn't a full 64 KB. */ - if (inputSize > 4 KB) { + if (inputSize >= LZ4_DICTCTX_COPY_THRESHOLD) { /* For compressing large blobs, it is faster to pay the setup * cost to copy the dictionary's tables into the active context, * so that the compression loop is only looking into one table. @@ -1765,8 +1907,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, } else { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); } - } else { /* small data <= 4 KB */ - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + } else { /* external dictionary in the current context */ + if (LZ4_isDictSmall(streamPtr)) { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); } else { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); @@ -1787,7 +1929,7 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* LZ4_renormDictT(streamPtr, srcSize); - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + if (LZ4_isDictSmall(streamPtr)) { result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); } else { result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); @@ -1811,8 +1953,9 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; - DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer); + DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, (void*)safeBuffer); + if (dictSize < 0) dictSize = 0; /* invalid : save nothing, like LZ4_saveDictHC() */ if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } @@ -1875,11 +2018,13 @@ LZ4_decompress_unsafe_generic( { const BYTE* ip = istart; BYTE* op = (BYTE*)ostart; - BYTE* const oend = ostart + decompressedSize; + BYTE* oend; const BYTE* const prefixStart = ostart - prefixSize; DEBUGLOG(5, "LZ4_decompress_unsafe_generic"); if (dictStart == NULL) assert(dictSize == 0); + if (decompressedSize < 0) return -1; + oend = ostart + decompressedSize; while (1) { /* start new sequence */ @@ -1964,42 +2109,32 @@ LZ4_decompress_unsafe_generic( /* Read the variable-length literal or match length. * - * @ip : input pointer - * @ilimit : position after which if length is not decoded, the input is necessarily corrupted. - * @initial_check - check ip >= ipmax before start of loop. Returns initial_error if so. - * @error (output) - error code. Must be set to 0 before call. + * @ipPtr : pointer to input pointer, will be advanced + * @ilimit : read is forbidden beyond this position (must be within input buffer) **/ typedef size_t Rvl_t; static const Rvl_t rvl_error = (Rvl_t)(-1); LZ4_FORCE_INLINE Rvl_t -read_variable_length(const BYTE** ip, const BYTE* ilimit, - int initial_check) +read_variable_length(const BYTE** ipPtr, const BYTE* ilimit) { Rvl_t s, length = 0; - assert(ip != NULL); - assert(*ip != NULL); + assert(ipPtr != NULL); + assert(*ipPtr != NULL); assert(ilimit != NULL); - if (initial_check && unlikely((*ip) >= ilimit)) { /* read limit reached */ + if (unlikely((*ipPtr) >= ilimit)) { /* read limit reached */ return rvl_error; } - s = **ip; - (*ip)++; + s = **ipPtr; + (*ipPtr)++; length += s; - if (unlikely((*ip) > ilimit)) { /* read limit reached */ - return rvl_error; - } - /* accumulator overflow detection (32-bit mode only) */ - if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { - return rvl_error; - } if (likely(s != 255)) return length; do { - s = **ip; - (*ip)++; - length += s; - if (unlikely((*ip) > ilimit)) { /* read limit reached */ + if (unlikely((*ipPtr) >= ilimit)) { /* read limit reached */ return rvl_error; } + s = **ipPtr; + (*ipPtr)++; + length += s; /* accumulator overflow detection (32-bit mode only) */ if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { return rvl_error; @@ -2009,6 +2144,52 @@ read_variable_length(const BYTE** ip, const BYTE* ilimit, return length; } +/* LZ4_copyMatchExact() : + * copy a match of length bytes from match to op (match < op, overlap allowed), + * writing nothing beyond op+length. Invalid offset 0 (match == op) produces 0 bytes. + * Not inlined : rarely used, its memcpy() calls would burden the decoder's hot loops. + * @return : output position after the match */ +LZ4_FORCE_NOINLINE BYTE* +LZ4_copyMatchExact(BYTE* op, const BYTE* match, size_t length) +{ + if (unlikely(match == op)) { /* invalid offset 0 : clear the match segment, see doc/lz4_Block_format.md */ + MEM_INIT(op, 0, length); + return op + length; + } + /* bytes from match to op repeat with period (op-match) : + * copy all of them, so the distance doubles at each round. + * Beyond 2 KB, keep the distance : larger copies of just written data are slower */ + while (length > (size_t)(op - match)) { + size_t const available = (size_t)(op - match); + LZ4_memcpy(op, match, available); + op += available; + length -= available; + if (available >= 2048) match += available; + } + LZ4_memcpy(op, match, length); + return op + length; +} + +/* LZ4_copyExtDictMatch() : + * copy a match starting within the external dictionary (match < lowPrefix), + * possibly stretching into the current block. + * @return : output position after the match */ +LZ4_FORCE_INLINE BYTE* +LZ4_copyExtDictMatch(BYTE* op, const BYTE* match, size_t length, + const BYTE* lowPrefix, const BYTE* dictEnd) +{ + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); + return op + length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + LZ4_memcpy(op, dictEnd - copySize, copySize); + return LZ4_copyMatchExact(op + copySize, lowPrefix, length - copySize); + } +} + /*! LZ4_decompress_generic() : * This generic decompression function covers all use cases. * It shall be instantiated several times, using different sets of directives. @@ -2020,7 +2201,7 @@ LZ4_decompress_generic( const char* const src, char* const dst, int srcSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ + int outputSize, /* dstCapacity, or MIN(targetOutputSize, dstCapacity) when partialDecoding */ earlyEnd_directive partialDecoding, /* full, partial */ dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ @@ -2036,21 +2217,21 @@ LZ4_decompress_generic( BYTE* op = (BYTE*) dst; BYTE* const oend = op + outputSize; - BYTE* cpy; const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; - const int checkOffset = (dictSize < (int)(64 KB)); + /* withPrefix64k : offset <= 65535 can't reach beyond the 64 KB prefix */ + const int checkOffset = (dict != withPrefix64k) && (dictSize < (int)(64 KB)); /* Set up the "end" pointers for the shortcut. */ const BYTE* const shortiend = iend - 14 /*maxLL*/ - 2 /*offset*/; const BYTE* const shortoend = oend - 14 /*maxLL*/ - 18 /*maxML*/; + /* shared by both loops : the fast loop can hand over a decoded match (safe_match_copy) */ const BYTE* match; size_t offset; - unsigned token; - size_t length; + size_t ml; /* match length */ DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); @@ -2060,9 +2241,10 @@ LZ4_decompress_generic( if (unlikely(outputSize==0)) { /* Empty output buffer */ if (partialDecoding) return 0; - return ((srcSize==1) && (*ip==0)) ? 0 : -1; + if ((srcSize==1) && (*ip==0)) return 0; /* empty block */ + return (srcSize > 1) ? -2 : -1; /* -2 : output buffer too small */ } - if (unlikely(srcSize==0)) { return -1; } + if (unlikely(srcSize<=0)) { return -1; } /* LZ4_FAST_DEC_LOOP: * designed for modern OoO performance cpus, @@ -2073,41 +2255,67 @@ LZ4_decompress_generic( DEBUGLOG(6, "move to safe decode loop"); goto safe_decode; } + /* Ring buffer restarting at its beginning : the oldest referenced history (max distance behind op) + * lies in the external dictionary, ahead of op, always at the same distance. + * Rings sized with the former LZ4_DECODER_RING_BUFFER_SIZE() (margin 14, up to v1.10.0) + * still work, but leave as little as 16 bytes : + * the fast loop's wide copies could overwrite it, the safe loop's can't. */ + if ( (dict==usingExtDict) + && ((uptrval)dictEnd - (uptrval)lowPrefix - LZ4_DISTANCE_ABSOLUTE_MAX < FASTLOOP_SAFE_DISTANCE) ) { + DEBUGLOG(6, "history just ahead : move to safe decode loop"); + goto safe_decode; + } - /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE */ + /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE, + * and input is not too close to iend. + * It hands over to the safe loop either at a sequence start (break), + * or once a match is fully decoded (goto safe_match_copy). */ DEBUGLOG(6, "using fast decode loop"); while (1) { + unsigned token; + size_t ll; /* literal length */ + BYTE* cpy; /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ assert(oend - op >= FASTLOOP_SAFE_DISTANCE); assert(ip < iend); token = *ip++; - length = token >> ML_BITS; /* literal length */ - DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + ll = token >> ML_BITS; + DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)ll); + + /* too close to input end : the safe loop restarts this sequence */ + if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { ip--; break; } /* decode literal length */ - if (length == RUN_MASK) { - size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (ll == RUN_MASK) { + /* literal length >= RUN_MASK means >= RUN_MASK literal bytes follow the extension bytes, + * so extension bytes cannot reach the last RUN_MASK bytes of input */ + const BYTE* const tokenPos = ip-1; + size_t const addl = read_variable_length(&ip, iend - RUN_MASK); if (addl == rvl_error) { DEBUGLOG(6, "error reading long literal length"); goto _output_error; } - length += addl; - if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ - if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + ll += addl; + if (unlikely((uptrval)(op)+ll<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+ll<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ - /* copy literals */ + /* copy literals, unless too close to the end for a wild copy : + * nothing is written yet, so the safe loop can restart this sequence */ + cpy = op+ll; LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); - if ((op+length>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } - LZ4_wildCopy32(op, ip, op+length); - ip += length; op += length; - } else if (ip <= iend-(16 + 1/*max lit + offset + nextToken*/)) { - /* We don't need to check oend, since we check it once for each loop below */ - DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); + #if LZ4_WILDCOPY64 + if ((cpy>oend-64) || (ip+ll>iend-64)) { ip = tokenPos; break; } + LZ4_wildCopy64(op, ip, cpy); + #else + if ((cpy>oend-32) || (ip+ll>iend-32)) { ip = tokenPos; break; } + LZ4_wildCopy32(op, ip, cpy); + #endif + ip += ll; op = cpy; + } else { + DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)ll); /* Literals can only be <= 14, but hope compilers optimize better when copy by a register size */ LZ4_memcpy(op, ip, 16); - ip += length; op += length; - } else { - goto safe_literal_copy; + ip += ll; op += ll; } /* get offset */ @@ -2117,86 +2325,64 @@ LZ4_decompress_generic( assert(match <= op); /* overflow check */ /* get matchlength */ - length = token & ML_MASK; - DEBUGLOG(7, " match length token = %u (len==%u)", (unsigned)length, (unsigned)length+MINMATCH); + ml = token & ML_MASK; + DEBUGLOG(7, " match length token = %u (len==%u)", (unsigned)ml, (unsigned)ml+MINMATCH); - if (length == ML_MASK) { - size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (ml == ML_MASK) { + /* after match length extension bytes, at least 1 token + LASTLITERALS literals must remain */ + size_t const addl = read_variable_length(&ip, iend - (1 + LASTLITERALS)); if (addl == rvl_error) { DEBUGLOG(5, "error reading long match length"); goto _output_error; } - length += addl; - length += MINMATCH; - DEBUGLOG(7, " long match length == %u", (unsigned)length); - if (unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ - if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { - goto safe_match_copy; - } - } else { - length += MINMATCH; - if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { - DEBUGLOG(7, "moving to safe_match_copy (ml==%u)", (unsigned)length); - goto safe_match_copy; - } + ml += addl; + DEBUGLOG(7, " long match length == %u", (unsigned)ml + MINMATCH); + if (unlikely((uptrval)(op)+ml<(uptrval)op)) { goto _output_error; } /* overflow detection */ + } + ml += MINMATCH; - /* Fastpath check: skip LZ4_wildCopy32 when true */ - if ((dict == withPrefix64k) || (match >= lowPrefix)) { - if (offset >= 8) { - assert(match >= lowPrefix); - assert(match <= op); - assert(op + 18 <= oend); + if (op + ml >= oend - FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(7, "moving to safe_match_copy (ml==%u)", (unsigned)ml); + goto safe_match_copy; + } - LZ4_memcpy(op, match, 8); - LZ4_memcpy(op+8, match+8, 8); - LZ4_memcpy(op+16, match+16, 2); - op += length; - continue; - } } } + /* match length from token only (<= 18), no overlap, within prefix : copy 18 bytes */ + if ( ((token & ML_MASK) != ML_MASK) && (offset >= 8) + && ((dict == withPrefix64k) || (match >= lowPrefix)) ) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + LZ4_memcpy(op, match, 8); + LZ4_memcpy(op+8, match+8, 8); + LZ4_memcpy(op+16, match+16, 2); + op += ml; + continue; + } + + /* match ends far from oend : no end-of-block condition to check */ + assert(op + ml < oend - FASTLOOP_SAFE_DISTANCE); if ( checkOffset && (unlikely(match + dictSize < lowPrefix)) ) { DEBUGLOG(5, "Error : pos=%zi, offset=%zi => outside buffers", op-lowPrefix, op-match); - goto _output_error; + goto _offset_error; } /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { assert(dictEnd != NULL); - if (unlikely(op+length > oend-LASTLITERALS)) { - if (partialDecoding) { - DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd"); - length = MIN(length, (size_t)(oend-op)); - } else { - DEBUGLOG(6, "end-of-block condition violated") - goto _output_error; - } } - - if (length <= (size_t)(lowPrefix-match)) { - /* match fits entirely within external dictionary : just copy */ - LZ4_memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match stretches into both external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix - match); - size_t const restSize = length - copySize; - LZ4_memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) { *op++ = *copyFrom++; } - } else { - LZ4_memcpy(op, lowPrefix, restSize); - op += restSize; - } } + op = LZ4_copyExtDictMatch(op, match, ml, lowPrefix, dictEnd); continue; } /* copy match within block */ - cpy = op + length; + cpy = op + ml; - assert((op <= oend) && (oend-op >= 32)); + assert((op <= oend) && (oend-op >= 64)); if (unlikely(offset<16)) { LZ4_memcpy_using_offset(op, match, cpy, offset); + #if LZ4_WILDCOPY64 + } else if (offset >= 64) { + LZ4_wildCopy64(op, match, cpy); + #endif } else { LZ4_wildCopy32(op, match, cpy); } @@ -2206,227 +2392,146 @@ LZ4_decompress_generic( safe_decode: #endif - /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + /* Safe loop : decode remaining sequences, without writing beyond oend nor reading beyond iend. + * With LZ4_FAST_DEC_LOOP, that's the end of the block (or a block too small for the fast loop), + * entered at a sequence start, or at safe_match_copy with a decoded match. + * Without it, that's the whole block. */ DEBUGLOG(6, "using safe decode loop"); while (1) { + unsigned token; + size_t ll; /* literal length */ + BYTE* cpy; assert(ip < iend); token = *ip++; - length = token >> ML_BITS; /* literal length */ - DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + ll = token >> ML_BITS; + DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)ll); /* A two-stage shortcut for the most common case: * 1) If the literal length is 0..14, and there is enough space, - * enter the shortcut and copy 16 bytes on behalf of the literals - * (in the fast mode, only 8 bytes can be safely copied this way). + * enter the shortcut and copy 16 bytes on behalf of the literals. * 2) Further if the match length is 4..18, copy 18 bytes in a similar * manner; but we ensure that there's enough space in the output for * those 18 bytes earlier, upon entering the shortcut (in other words, * there is a combined check for both stages). */ - if ( (length != RUN_MASK) + if ( (ll != RUN_MASK) /* strictly "less than" on input, to re-enter the loop with at least one byte */ && likely((ip < shortiend) & (op <= shortoend)) ) { /* Copy the literals */ LZ4_memcpy(op, ip, 16); - op += length; ip += length; + op += ll; ip += ll; /* The second stage: prepare for match copying, decode full info. * If it doesn't work out, the info won't be wasted. */ - length = token & ML_MASK; /* match length */ - DEBUGLOG(7, "blockPos%6u: matchLength token = %u (len=%u)", (unsigned)(op-(BYTE*)dst), (unsigned)length, (unsigned)length + 4); offset = LZ4_readLE16(ip); ip += 2; match = op - offset; assert(match <= op); /* check overflow */ + ml = token & ML_MASK; + DEBUGLOG(7, "blockPos%6u: matchLength token = %u (len=%u)", (unsigned)(op-(BYTE*)dst), (unsigned)ml, (unsigned)ml + 4); /* Do not deal with overlapping matches. */ - if ( (length != ML_MASK) + if ( (ml != ML_MASK) && (offset >= 8) && (dict==withPrefix64k || match >= lowPrefix) ) { /* Copy the match. */ LZ4_memcpy(op + 0, match + 0, 8); LZ4_memcpy(op + 8, match + 8, 8); LZ4_memcpy(op +16, match +16, 2); - op += length + MINMATCH; + op += ml + MINMATCH; /* Both stages worked, load the next token. */ continue; } + /* The second stage didn't work out, but the info is ready for the match copy below. */ + } else { + /* decode literal length */ + if (ll == RUN_MASK) { + /* literal length >= RUN_MASK means >= RUN_MASK literal bytes follow the extension bytes, + * so extension bytes cannot reach the last RUN_MASK bytes of input */ + size_t const addl = read_variable_length(&ip, iend-RUN_MASK); + if (addl == rvl_error) { goto _output_error; } + ll += addl; + if (unlikely((uptrval)(op)+ll<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+ll<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + } - /* The second stage didn't work out, but the info is ready. - * Propel it right to the point of match copying. */ - goto _copy_match; - } - - /* decode literal length */ - if (length == RUN_MASK) { - size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); - if (addl == rvl_error) { goto _output_error; } - length += addl; - if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ - if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ - } - -#if LZ4_FAST_DEC_LOOP - safe_literal_copy: -#endif - /* copy literals */ - cpy = op+length; - - LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); - if ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) { - /* We've either hit the input parsing restriction or the output parsing restriction. - * In the normal scenario, decoding a full block, it must be the last sequence, - * otherwise it's an error (invalid input or dimensions). - * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow. - */ - if (partialDecoding) { - /* Since we are partial decoding we may be in this block because of the output parsing - * restriction, which is not valid since the output buffer is allowed to be undersized. - */ - DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end") - DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length); - DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op)); - DEBUGLOG(7, "partialDecoding: remaining space in srcBuffer : %i", (int)(iend - ip)); - /* Finishing in the middle of a literals segment, - * due to lack of input. - */ - if (ip+length > iend) { - length = (size_t)(iend-ip); - cpy = op + length; - } - /* Finishing in the middle of a literals segment, - * due to lack of output space. - */ - if (cpy > oend) { - cpy = oend; - assert(op<=oend); - length = (size_t)(oend-op); - } - } else { - /* We must be on the last sequence (or invalid) because of the parsing limitations - * so check that we exactly consume the input and don't overrun the output buffer. - */ - if ((ip+length != iend) || (cpy > oend)) { - DEBUGLOG(5, "should have been last run of literals") - DEBUGLOG(5, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend); - DEBUGLOG(5, "or cpy(%p) > (oend-MFLIMIT)(%p)", cpy, oend-MFLIMIT); - DEBUGLOG(5, "after writing %u bytes / %i bytes available", (unsigned)(op-(BYTE*)dst), outputSize); + /* copy literals */ + cpy = op+ll; + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ((cpy>oend-MFLIMIT) || (ip+ll>iend-(2+1+LASTLITERALS))) { + /* close to input or output end : these must be the last literals of the block, + * except with partialDecoding, which can also stop within them */ + if (partialDecoding) { + /* stop at input end, or when output is full */ + ll = MIN(ll, (size_t)(iend-ip)); + ll = MIN(ll, (size_t)(oend-op)); + } else if ((ip+ll != iend) || (cpy > oend)) { + /* last literals must end exactly at input end, within output */ + DEBUGLOG(5, "invalid last literals : they end at iend%+zi and oend%+zi", (ip+ll)-iend, cpy-oend); + if ((ip+ll <= iend) && (cpy > oend-MFLIMIT)) goto _dst_too_small; /* output full, input left */ goto _output_error; } + LZ4_memmove(op, ip, ll); /* supports overlapping memory regions, for in-place decompression scenarios */ + ip += ll; + op += ll; + /* end of block, unless partialDecoding can go on : output not full, input left for a match */ + if (!partialDecoding || (op == oend) || (ip >= (iend-2))) { + break; + } + } else { + LZ4_wildCopy8(op, ip, cpy); /* can overwrite up to 8 bytes beyond cpy */ + ip += ll; op = cpy; } - LZ4_memmove(op, ip, length); /* supports overlapping memory regions, for in-place decompression scenarios */ - ip += length; - op += length; - /* Necessarily EOF when !partialDecoding. - * When partialDecoding, it is EOF if we've either - * filled the output buffer or - * can't proceed with reading an offset for following match. - */ - if (!partialDecoding || (cpy == oend) || (ip >= (iend-2))) { - break; - } - } else { - LZ4_wildCopy8(op, ip, cpy); /* can overwrite up to 8 bytes beyond cpy */ - ip += length; op = cpy; + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + + /* get matchlength */ + ml = token & ML_MASK; + DEBUGLOG(7, "blockPos%6u: matchLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)ml); } - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; - - /* get matchlength */ - length = token & ML_MASK; - DEBUGLOG(7, "blockPos%6u: matchLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); - - _copy_match: - if (length == ML_MASK) { - size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + /* literals copied, offset decoded, ml == match length token */ + if (ml == ML_MASK) { + /* after match length extension bytes, at least 1 token + LASTLITERALS literals must remain */ + size_t const addl = read_variable_length(&ip, iend - (1 + LASTLITERALS)); if (addl == rvl_error) { goto _output_error; } - length += addl; - if (unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + ml += addl; + if (unlikely((uptrval)(op)+ml<(uptrval)op)) goto _output_error; /* overflow detection */ } - length += MINMATCH; + ml += MINMATCH; #if LZ4_FAST_DEC_LOOP safe_match_copy: #endif - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + /* literals copied, match fully decoded (ml includes MINMATCH), offset not validated yet */ + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _offset_error; /* Error : offset outside buffers */ + + /* end of block : the last LASTLITERALS bytes must be literals */ + assert(op <= oend); + if (unlikely(op+ml > oend-LASTLITERALS)) { + if (partialDecoding) ml = MIN(ml, (size_t)(oend-op)); /* partialDecoding : may end anywhere */ + else goto _dst_too_small; + } + /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { assert(dictEnd != NULL); - if (unlikely(op+length > oend-LASTLITERALS)) { - if (partialDecoding) length = MIN(length, (size_t)(oend-op)); - else goto _output_error; /* doesn't respect parsing restriction */ - } - - if (length <= (size_t)(lowPrefix-match)) { - /* match fits entirely within external dictionary : just copy */ - LZ4_memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match stretches into both external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix - match); - size_t const restSize = length - copySize; - LZ4_memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } else { - LZ4_memcpy(op, lowPrefix, restSize); - op += restSize; - } } + op = LZ4_copyExtDictMatch(op, match, ml, lowPrefix, dictEnd); continue; } assert(match >= lowPrefix); - /* copy match within block */ - cpy = op + length; - - /* partialDecoding : may end anywhere within the block */ - assert(op<=oend); - if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { - size_t const mlen = MIN(length, (size_t)(oend-op)); - const BYTE* const matchEnd = match + mlen; - BYTE* const copyEnd = op + mlen; - if (matchEnd > op) { /* overlap copy */ - while (op < copyEnd) { *op++ = *match++; } - } else { - LZ4_memcpy(op, match, mlen); - } - op = copyEnd; - if (op == oend) { break; } + /* close to block end : no wild copy */ + if (unlikely(op+ml > oend-MATCH_SAFEGUARD_DISTANCE)) { + op = LZ4_copyMatchExact(op, match, ml); + if (op == oend) { break; } /* partialDecoding : output buffer full */ continue; } - if (unlikely(offset<8)) { - LZ4_write32(op, 0); /* silence msan warning when offset==0 */ - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += inc32table[offset]; - LZ4_memcpy(op+4, match, 4); - match -= dec64table[offset]; - } else { - LZ4_memcpy(op, match, 8); - match += 8; - } - op += 8; - - if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { - BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy8(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; - } - while (op < cpy) { *op++ = *match++; } - } else { - LZ4_memcpy(op, match, 8); - if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } - } + /* copy match within block */ + cpy = op + ml; + LZ4_memcpy_using_offset_base(op, match, cpy, offset); op = cpy; /* wildcopy correction */ } @@ -2434,9 +2539,13 @@ LZ4_decompress_generic( DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - /* Overflow error detected */ + /* errors : see LZ4_decompress_safe() in lz4.h */ _output_error: - return (int) (-(((const char*)ip)-src))-1; + return -1; + _dst_too_small: + return -2; + _offset_error: + return -3; } } @@ -2585,6 +2694,7 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + if (dictSize < 0) return 0; lz4sd->prefixSize = (size_t)dictSize; if (dictSize) { assert(dictionary != NULL); @@ -2610,6 +2720,10 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti */ int LZ4_decoderRingBufferSize(int maxBlockSize) { + /* the margin must cover the widest write beyond a copy of the safe loop (16 bytes), minus 2 : + * LZ4_decompress_generic() avoids the fast loop when history is closer than FASTLOOP_SAFE_DISTANCE, + * which the current margin (64) never is. The former margin (14) remains valid. */ + LZ4_STATIC_ASSERT(LZ4_DECODER_RING_BUFFER_SIZE(0) - 65536 >= 16 - 2); if (maxBlockSize < 0) return 0; if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; if (maxBlockSize < 16) maxBlockSize = 16; @@ -2714,6 +2828,7 @@ Advanced decoding functions : int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { + if (dictSize < 0) return -1; if (dictSize==0) return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (dictStart+dictSize == dest) { @@ -2729,6 +2844,7 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, const char* dictStart, int dictSize) { + if (dictSize < 0) return -1; if (dictSize==0) return LZ4_decompress_safe_partial(source, dest, compressedSize, targetOutputSize, dstCapacity); if (dictStart+dictSize == dest) { @@ -2744,6 +2860,7 @@ int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int co int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { + if (dictSize < 0) return -1; if (dictSize==0 || dictStart+dictSize == dest) return LZ4_decompress_unsafe_generic( (const BYTE*)source, (BYTE*)dest, originalSize, diff --git a/public/common/tracy_lz4.hpp b/public/common/tracy_lz4.hpp index 5b29fdb8..e31fb961 100644 --- a/public/common/tracy_lz4.hpp +++ b/public/common/tracy_lz4.hpp @@ -1,7 +1,7 @@ /* * LZ4 - Fast LZ compression algorithm * Header File - * Copyright (C) 2011-2023, Yann Collet. + * Copyright (c) Yann Collet. All rights reserved. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -32,15 +32,15 @@ - LZ4 homepage : http://www.lz4.org - LZ4 source repository : https://github.com/lz4/lz4 */ - -#ifndef TRACY_LZ4_H_2983827168210 -#define TRACY_LZ4_H_2983827168210 - /* --- Dependency --- */ #include /* size_t */ #include +#ifndef TRACY_LZ4_H_2983827168210 +#define TRACY_LZ4_H_2983827168210 + + /** Introduction @@ -105,7 +105,7 @@ * LZ4_memcpy, LZ4_memmove, LZ4_memset. * - It only enables LZ4/HC functions which don't use heap. * All LZ4F_* functions are not supported. - * - See tests/freestanding.c to check its basic setup. + * - See tests/compat/freestanding.c to check its basic setup. */ #if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) # define LZ4_HEAPMODE 0 @@ -127,7 +127,7 @@ /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 10 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_MINOR 11 /* for new (non-breaking) interface capabilities */ #define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) @@ -195,9 +195,12 @@ LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int * @compressedSize : is the exact complete size of the compressed block. * @dstCapacity : is the size of destination buffer (which must be already allocated), * presumed an upper bound of decompressed size. - * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) - * If destination buffer is not large enough, decoding will stop and output an error code (negative value). - * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity), + * or a negative error code (v1.11.0+) : + * -1 : generic error : malformed or truncated input, or invalid parameters + * -2 : decoded data doesn't fit into dstCapacity : buffer too small, or corrupted input + * -3 : data references bytes before the start of output : missing or wrong dictionary, or corrupted input + * Any negative value is an error : more specific codes may be added in future versions. * Note 1 : This function is protected against malicious data packets : * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, * even if the compressed block is maliciously modified to order the decoder to do these actions. @@ -231,8 +234,8 @@ LZ4LIB_API int LZ4_compressBound(int inputSize); The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c). - Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c). + Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see LZ4_STATIC_LINKING_ONLY section). + Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see LZ4_STATIC_LINKING_ONLY section). */ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -259,10 +262,12 @@ LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* d * @return : Nb bytes written into 'dst' (necessarily <= dstCapacity) * or 0 if compression fails. * - * Note : from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+): - * the produced compressed content could, in specific circumstances, - * require to be decompressed into a destination buffer larger - * by at least 1 byte than the content to decompress. + * Note : 'targetDstSize' must be >= 1, because it's the smallest valid lz4 payload. + * + * Note 2:from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+): + * the produced compressed content could, in rare circumstances, + * require to be decompressed into a destination buffer + * larger by at least 1 byte than decompressesSize. * If an application uses `LZ4_compress_destSize()`, * it's highly recommended to update liblz4 to v1.9.2 or better. * If this can't be done or ensured, @@ -281,7 +286,7 @@ LZ4LIB_API int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr * whenever only the beginning of a block is required. * * @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize) - * If source stream is detected malformed, function returns a negative result. + * or a negative error code (see LZ4_decompress_safe()), never -2. * * Note 1 : @return can be < targetOutputSize, if compressed block contains less data. * @@ -489,7 +494,7 @@ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const * or 0 if there is an error (invalid maxBlockSize). */ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); -#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 64 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ /*! LZ4_decompress_safe_continue() : * This decoding function allows decompression of consecutive blocks in "streaming" mode. @@ -504,8 +509,7 @@ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); * @dstCapacity : size of destination buffer (which must be already allocated), * must be an upper bound of decompressed size. * @return : number of bytes decompressed into destination buffer (necessarily <= dstCapacity) - * If destination buffer is not large enough, decoding will stop and output an error code (negative value). - * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * or a negative error code (see LZ4_decompress_safe()). * * The last 64KB of previously decoded data *must* remain available and unmodified * at the memory position where they were previously decoded. @@ -519,9 +523,10 @@ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); * - Synchronized mode : * Decompression buffer size is _exactly_ the same as compression buffer size, * and follows exactly same update rule (block boundaries at same positions), - * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), + * and decoding function is provided with exact decompressed size of each block, including the last one + * (decoding may write up to dst + dstCapacity, over data still referenced), * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). - * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. + * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize + 64 bytes. * In which case, encoding and decoding buffers do not need to be synchronized, * and encoding ring buffer can have any size, including small ones ( < 64 KB). * @@ -600,6 +605,14 @@ LZ4_decompress_safe_partial_usingDict(const char* src, char* dst, # define LZ4LIB_STATIC_API #endif +/*! LZ4_ACCELERATION_DEFAULT, LZ4_ACCELERATION_MAX : + * "acceleration" values <= 0 are replaced by LZ4_ACCELERATION_DEFAULT. + * Values > LZ4_ACCELERATION_MAX are replaced by LZ4_ACCELERATION_MAX : + * higher values make no difference (fix #876). + */ +#define LZ4_ACCELERATION_DEFAULT 1 +#define LZ4_ACCELERATION_MAX 65537 + namespace tracy { @@ -611,7 +624,7 @@ namespace tracy * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). * From a high level, the difference is that * this function initializes the provided state with a call to something like LZ4_resetStream_fast() - * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). + * while LZ4_compress_fast_extState() starts with a call to LZ4_initStream(). */ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); @@ -707,10 +720,10 @@ namespace tracy #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) - typedef int8_t LZ4_i8; - typedef uint8_t LZ4_byte; - typedef uint16_t LZ4_u16; - typedef uint32_t LZ4_u32; + typedef int8_t LZ4_i8; + typedef unsigned char LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; #else typedef signed char LZ4_i8; typedef unsigned char LZ4_byte; @@ -811,36 +824,9 @@ union LZ4_streamDecode_u { # endif #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ -/*! Obsolete compression functions (since v1.7.3) */ -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); - -/*! Obsolete decompression functions (since v1.8.0) */ -LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); -LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); - -/* Obsolete streaming functions (since v1.7.0) - * degraded functionality; do not use! - * - * In order to perform streaming compression, these functions depended on data - * that is no longer tracked in the state. They have been preserved as well as - * possible: using them will still produce a correct output. However, they don't - * actually retain any history between compression calls. The compression ratio - * achieved will therefore be no better than compressing each chunk - * independently. - */ -LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); -LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); -LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); -LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); - -/*! Obsolete streaming decoding functions (since v1.7.0) */ -LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); -LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); +/*! Removed in v1.11.0, deprecated since v1.7.x - v1.8.0 (liblz4 still exports them, for linked programs) : + * LZ4_compress*(), LZ4_uncompress*(), LZ4_create(), LZ4_sizeofStreamState(), LZ4_resetStreamState(), + * LZ4_slideInputBuffer(), LZ4_decompress_safe_withPrefix64k(), LZ4_decompress_fast_withPrefix64k() */ /*! Obsolete LZ4_decompress_fast variants (since v1.9.0) : * These functions used to be faster than LZ4_decompress_safe(), @@ -876,13 +862,11 @@ LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecod LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); -/*! LZ4_resetStream() : - * An LZ4_stream_t structure must be initialized at least once. - * This is done with LZ4_initStream(), or LZ4_resetStream(). - * Consider switching to LZ4_initStream(), - * invoking LZ4_resetStream() will trigger deprecation warnings in the future. +/*! LZ4_resetStream() : deprecated (v1.11.0) + * Same as LZ4_initStream(streamPtr, sizeof(LZ4_stream_t)) : prefer LZ4_initStream(). + * To start a new stream with an initialized state, LZ4_resetStream_fast() is faster. */ -LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); +LZ4_DEPRECATED("use LZ4_initStream() instead") LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); } diff --git a/public/common/tracy_lz4hc.cpp b/public/common/tracy_lz4hc.cpp index eaf5818a..af07130f 100644 --- a/public/common/tracy_lz4hc.cpp +++ b/public/common/tracy_lz4hc.cpp @@ -1,6 +1,6 @@ /* LZ4 HC - High Compression Mode of LZ4 - Copyright (C) 2011-2020, Yann Collet. + Copyright (c) Yann Collet. All rights reserved. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -81,7 +81,9 @@ typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; /*=== Macros ===*/ +#undef MIN #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#undef MAX #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) @@ -91,21 +93,24 @@ typedef struct { lz4hc_strat_e strat; int nbSearches; U32 targetLength; + int nbSearchesSeq; /* lz4opt with favorDecSpeed only : budget at positions only worth searching to save a sequence */ + int dictCopyMin; /* attached dictionary : from this input size, its tables (256 KB) are copied rather than referenced (dictCtx), + * which compresses as well or better, and pays off sooner at levels which search more */ } cParams_t; static const cParams_t k_clTable[LZ4HC_CLEVEL_MAX+1] = { - { lz4mid, 2, 16 }, /* 0, unused */ - { lz4mid, 2, 16 }, /* 1, unused */ - { lz4mid, 2, 16 }, /* 2 */ - { lz4hc, 4, 16 }, /* 3 */ - { lz4hc, 8, 16 }, /* 4 */ - { lz4hc, 16, 16 }, /* 5 */ - { lz4hc, 32, 16 }, /* 6 */ - { lz4hc, 64, 16 }, /* 7 */ - { lz4hc, 128, 16 }, /* 8 */ - { lz4hc, 256, 16 }, /* 9 */ - { lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/ - { lz4opt, 512,128 }, /*11 */ - { lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ + { lz4mid, 2, 16, 0, 4 KB }, /* 0, unused */ + { lz4mid, 2, 16, 0, 4 KB }, /* 1, unused */ + { lz4mid, 2, 16, 0, 4 KB }, /* 2 */ + { lz4hc, 4, 16, 0, 4 KB }, /* 3 */ + { lz4hc, 8, 16, 0, 4 KB }, /* 4 */ + { lz4hc, 16, 16, 0, 4 KB }, /* 5 */ + { lz4hc, 32, 16, 0, 4 KB }, /* 6 */ + { lz4hc, 64, 16, 0, 4 KB }, /* 7 */ + { lz4hc, 128, 16, 0, 4 KB }, /* 8 */ + { lz4hc, 256, 16, 0, 4 KB }, /* 9 */ + { lz4opt, 96, 64, 1, 4 KB }, /*10==LZ4HC_CLEVEL_OPT_MIN*/ + { lz4opt, 512,128, 2, 2 KB }, /*11 */ + { lz4opt,16384,LZ4_OPT_NUM, 64, 1 KB }, /* 12==LZ4HC_CLEVEL_MAX */ }; static cParams_t LZ4HC_getCLevelParams(int cLevel) @@ -265,9 +270,33 @@ static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start) /************************************** * Encode **************************************/ +#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2) +# define RAWLOG(...) fprintf(stderr, __VA_ARGS__) +void LZ4HC_hexOut(const void* src, size_t len) +{ + const BYTE* p = (const BYTE*)src; + size_t n; + for (n=0; n= _lev) { \ + RAWLOG("match bytes: "); \ + LZ4HC_hexOut(_ptr, _len); \ + RAWLOG("ref bytes: "); \ + LZ4HC_hexOut(_ref, _len); \ + } + +#else +# define HEX_CMP(l,p,r,_l) +#endif + /* LZ4HC_encodeSequence() : * @return : 0 if ok, - * 1 if buffer issue detected */ + * 1 if buffer issue detected : @ip, @op and @anchor are then unchanged */ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( const BYTE** _ip, BYTE** _op, @@ -281,47 +310,50 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( #define op (*_op) #define anchor (*_anchor) - size_t length; BYTE* const token = op++; #if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6) static const BYTE* start = NULL; static U32 totalCost = 0; - U32 const pos = (start==NULL) ? 0 : (U32)(anchor - start); + U32 const pos = (start==NULL) ? 0 : (U32)(anchor - start); /* only works for single segment */ U32 const ll = (U32)(ip - anchor); U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0; U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0; U32 const cost = 1 + llAdd + ll + 2 + mlAdd; if (start==NULL) start = anchor; /* only works for single segment */ - /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */ DEBUGLOG(6, "pos:%7u -- literals:%4u, match:%4i, offset:%5i, cost:%4u + %5u", pos, (U32)(ip - anchor), matchLength, offset, cost, totalCost); +# if 1 /* only works on single segment data */ + HEX_CMP(7, ip, ip-offset, matchLength); +# endif totalCost += cost; #endif /* Encode Literal length */ - length = (size_t)(ip - anchor); - LZ4_STATIC_ASSERT(notLimited == 0); - /* Check output limit */ - if (limit && ((op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) { - DEBUGLOG(6, "Not enough room to write %i literals (%i bytes remaining)", - (int)length, (int)(oend - op)); - return 1; - } - if (length >= RUN_MASK) { - size_t len = length - RUN_MASK; - *token = (RUN_MASK << ML_BITS); - for(; len >= 255 ; len -= 255) *op++ = 255; - *op++ = (BYTE)len; - } else { - *token = (BYTE)(length << ML_BITS); - } + { size_t litLen = (size_t)(ip - anchor); + LZ4_STATIC_ASSERT(notLimited == 0); + /* Check output limit */ + if (limit && ((op + (litLen / 255) + litLen + (2 + 1 + LASTLITERALS)) > oend)) { + DEBUGLOG(6, "Not enough room to write %i literals (%i bytes remaining)", + (int)litLen, (int)(oend - op)); + op = token; + return 1; + } + if (litLen >= RUN_MASK) { + size_t len = litLen - RUN_MASK; + *token = (RUN_MASK << ML_BITS); + for(; len >= 255 ; len -= 255) *op++ = 255; + *op++ = (BYTE)len; + } else { + *token = (BYTE)(litLen << ML_BITS); + } - /* Copy Literals */ - LZ4_wildCopy8(op, anchor, op + length); - op += length; + /* Copy Literals */ + LZ4_wildCopy8(op, anchor, op + litLen); + op += litLen; + } /* Encode Offset */ assert(offset <= LZ4_DISTANCE_MAX ); @@ -330,20 +362,21 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( /* Encode MatchLength */ assert(matchLength >= MINMATCH); - length = (size_t)matchLength - MINMATCH; - if (limit && (op + (length / 255) + (1 + LASTLITERALS) > oend)) { - DEBUGLOG(6, "Not enough room to write match length"); - return 1; /* Check output limit */ - } - if (length >= ML_MASK) { - *token += ML_MASK; - length -= ML_MASK; - for(; length >= 510 ; length -= 510) { *op++ = 255; *op++ = 255; } - if (length >= 255) { length -= 255; *op++ = 255; } - *op++ = (BYTE)length; - } else { - *token += (BYTE)(length); - } + { size_t mlCode = (size_t)matchLength - MINMATCH; + if (limit && (op + (mlCode / 255) + (1 + LASTLITERALS) > oend)) { + DEBUGLOG(6, "Not enough room to write match length"); + op = token; + return 1; + } + if (mlCode >= ML_MASK) { + *token += ML_MASK; + mlCode -= ML_MASK; + for(; mlCode >= 510 ; mlCode -= 510) { *op++ = 255; *op++ = 255; } + if (mlCode >= 255) { mlCode -= 255; *op++ = 255; } + *op++ = (BYTE)mlCode; + } else { + *token += (BYTE)(mlCode); + } } /* Prepare next loop */ ip += matchLength; @@ -356,6 +389,97 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( #undef anchor } +/* LZ4HC_encodeLastLiterals() : + * Encode the final run of literals, from @anchor to @iend. + * In fillOutput mode, the run may be truncated, to fit into dst. + * @oend : same as LZ4HC_encodeSequence() : end of dst, minus LASTLITERALS in fillOutput mode + * @return : 0 if ok, + * 1 if dst is too small (limitedOutput) */ +static int LZ4HC_encodeLastLiterals ( + const BYTE** _ip, + BYTE** _op, + const BYTE** _anchor, + const BYTE* iend, + limitedOutput_directive limit, + BYTE* oend) +{ +#define ip (*_ip) +#define op (*_op) +#define anchor (*_anchor) + + BYTE* const dstEnd = (limit == fillOutput) ? oend + LASTLITERALS : oend; + size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + llAdd + lastRunSize; + if (limit && (op + totalSize > dstEnd)) { + if (limit == limitedOutput) return 1; /* not enough space in dst */ + /* adapt lastRunSize to fill dst */ + lastRunSize = (size_t)(dstEnd - op) - 1 /*token*/; + llAdd = (lastRunSize + 256 - RUN_MASK) / 256; + lastRunSize -= llAdd; + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); + ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + assert(lastRunSize <= (size_t)(dstEnd - op)); + LZ4_memcpy(op, anchor, lastRunSize); + op += lastRunSize; + return 0; + +#undef ip +#undef op +#undef anchor +} + +/* LZ4HC_encodeLastSequence_fillOutput() : + * fillOutput mode only : current sequence doesn't fit into dst. + * Encode it anyway if all its literals fit, shortening its match if need be, + * but only if it leaves enough room for the final run of literals (MFLIMIT). + * @op : start of this sequence (before the failed LZ4HC_encodeSequence()) + * @oend : same as LZ4HC_encodeSequence() : end of dst minus LASTLITERALS */ +static void LZ4HC_encodeLastSequence_fillOutput ( + const BYTE** _ip, + BYTE** _op, + const BYTE** _anchor, + int matchLength, + int offset, + BYTE* oend) +{ +#define ip (*_ip) +#define op (*_op) +#define anchor (*_anchor) + + BYTE* const dstEnd = oend + LASTLITERALS; + size_t const ll = (size_t)(ip - anchor); + size_t const ll_addbytes = (ll + 240) / 255; + size_t const ll_totalCost = 1 + ll_addbytes + ll; + BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ + DEBUGLOG(6, "Last sequence is overflowing : %u literals, %u remaining space", + (unsigned)ll, (unsigned)(oend - op)); + if (op + ll_totalCost <= maxLitPos) { + /* ll validated; now adjust match length */ + size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); + size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); + assert(maxMlSize < INT_MAX); assert(matchLength >= 0); + if ((size_t)matchLength > maxMlSize) matchLength = (int)maxMlSize; + if (dstEnd - (op + ll_totalCost + 2) - 1 + matchLength >= MFLIMIT) { + DEBUGLOG(6, "Let's encode a last sequence (ll=%u, ml=%i)", (unsigned)ll, matchLength); + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), matchLength, offset, notLimited, oend); + } } + +#undef ip +#undef op +#undef anchor +} + typedef struct { int off; @@ -363,7 +487,7 @@ typedef struct { int back; /* negative value */ } LZ4HC_match_t; -LZ4HC_match_t LZ4HC_searchExtDict(const BYTE* ip, U32 ipIndex, +static LZ4HC_match_t LZ4HC_searchExtDict(const BYTE* ip, U32 ipIndex, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const LZ4HC_CCtx_internal* dictCtx, U32 gDictEndIndex, int currentBestML, int nbAttempts) @@ -472,6 +596,37 @@ static LZ4HC_match_t LZ4MID_searchExtDict(const BYTE* ip, U32 ipIndex, } } +/* LZ4HC_dropPositions() : + * In fillOutput mode, compression can stop at @start, before positions the parser has already referenced. + * The next block compresses them : it must not find them as matches. + * Remove their references from the hash table, up to @end (excluded). + * Costs one pass on these positions, or on the hash table, whichever is shorter. */ +static void +LZ4HC_dropPositions(LZ4HC_CCtx_internal* ctx, const BYTE* start, const BYTE* end, lz4hc_strat_e strat) +{ + U32* const hashTable = ctx->hashTable; + U32 const startIdx = ctx->dictLimit + (U32)(start - ctx->prefixStart); + if (end - start > LZ4HC_HASHTABLESIZE) { + int i; + for (i = 0; i < LZ4HC_HASHTABLESIZE; i++) { + /* branchless, so that it can be vectorized */ + hashTable[i] = (hashTable[i] >= startIdx) ? 0 : hashTable[i]; + } + } else { + U32 idx = startIdx; + const BYTE* p; + for (p = start; p < end; p++, idx++) { + if (strat == lz4mid) { + U32 const h4 = LZ4MID_hash4Ptr(p); + U32 const h8 = LZ4MID_HASHTABLESIZE + LZ4MID_hash8Ptr(p); + if (hashTable[h4] == idx) hashTable[h4] = 0; + if (hashTable[h8] == idx) hashTable[h8] = 0; + } else { + U32 const h = LZ4HC_hashPtr(p); + if (hashTable[h] == idx) hashTable[h] = 0; + } } } +} + /************************************** * Mid Compression (level 2) **************************************/ @@ -522,6 +677,34 @@ static LZ4MID_searchIntoDict_f select_searchDict_function(const LZ4HC_CCtx_inter return LZ4MID_searchHCDict; } +/* Length of the match between @ip and the candidate at index @pos (in prefix or extDict), + * or 0 if the candidate is out of reach */ +LZ4_FORCE_INLINE unsigned +LZ4MID_countMatch(const BYTE* ip, U32 ipIndex, U32 pos, const BYTE* matchlimit, + const BYTE* prefixPtr, U32 prefixIdx, + const BYTE* dictStart, U32 dictIdx) +{ + if (ipIndex - pos > LZ4_DISTANCE_MAX) return 0; + if (pos >= prefixIdx) { + const BYTE* const matchPtr = prefixPtr + (pos - prefixIdx); + assert(matchPtr < ip); + return LZ4_count(ip, matchPtr, matchlimit); + } + if (pos >= dictIdx) { + /* extDict : the match stops at the end of the dictionary */ + const BYTE* const matchPtr = dictStart + (pos - dictIdx); + size_t const safeLen = MIN(prefixIdx - pos, (size_t)(matchlimit - ip)); + return LZ4_count(ip, matchPtr, ip + safeLen); + } + return 0; +} + +/* preconditions: + * - *srcSizePtr within [1, LZ4_MAX_INPUT_SIZE] + * - src is valid + * - maxOutputSize >= 1 + * - dst is valid + */ static int LZ4MID_compress ( LZ4HC_CCtx_internal* const ctx, const char* const src, @@ -537,159 +720,119 @@ static int LZ4MID_compress ( const BYTE* ip = (const BYTE*)src; const BYTE* anchor = ip; const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - const BYTE* const ilimit = (iend - LZ4MID_HASHSIZE); + const BYTE* mflimit; /* set after the size check : would precede a small input */ + const BYTE* matchlimit; + const BYTE* ilimit; BYTE* op = (BYTE*)dst; - BYTE* oend = op + maxOutputSize; + /* fillOutput : keep room for the final run of literals (LZ4 format restriction) */ + BYTE* const oend = op + maxOutputSize - ((limit == fillOutput) ? LASTLITERALS : 0); const BYTE* const prefixPtr = ctx->prefixStart; const U32 prefixIdx = ctx->dictLimit; - const U32 ilimitIdx = (U32)(ilimit - prefixPtr) + prefixIdx; + U32 ilimitIdx; const BYTE* const dictStart = ctx->dictStart; const U32 dictIdx = ctx->lowLimit; const U32 gDictEndIndex = ctx->lowLimit; const LZ4MID_searchIntoDict_f searchIntoDict = (dict == usingDictCtxHc) ? select_searchDict_function(ctx->dictCtx) : NULL; - unsigned matchLength; - unsigned matchDistance; + const BYTE* refEnd = ip; /* fillOutput : the hash tables reference positions before MAX(refEnd, ip) */ - /* input sanitization */ DEBUGLOG(5, "LZ4MID_compress (%i bytes)", *srcSizePtr); + + /* preconditions verifications */ if (dict == usingDictCtxHc) DEBUGLOG(5, "usingDictCtxHc"); - assert(*srcSizePtr >= 0); - if (*srcSizePtr) assert(src != NULL); - if (maxOutputSize) assert(dst != NULL); - if (*srcSizePtr < 0) return 0; /* invalid */ - if (maxOutputSize < 0) return 0; /* invalid */ - if (*srcSizePtr > LZ4_MAX_INPUT_SIZE) { - /* forbidden: no input is allowed to be that large */ - return 0; - } - if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + assert(*srcSizePtr > 0); + assert(*srcSizePtr <= LZ4_MAX_INPUT_SIZE); + assert(src != NULL); + assert(maxOutputSize >= 1); + assert(dst != NULL); + if (*srcSizePtr < LZ4_minLength) goto _lz4mid_last_literals; /* Input too small, no compression (all literals) */ + mflimit = iend - MFLIMIT; + matchlimit = (iend - LASTLITERALS); + ilimit = (iend - LZ4MID_HASHSIZE); + ilimitIdx = (U32)(ilimit - prefixPtr) + prefixIdx; /* main loop */ while (ip <= mflimit) { const U32 ipIndex = (U32)(ip - prefixPtr) + prefixIdx; + unsigned matchLength; + unsigned matchDistance; /* search long match */ { U32 const h8 = LZ4MID_hash8Ptr(ip); U32 const pos8 = hash8Table[h8]; assert(h8 < LZ4MID_HASHTABLESIZE); assert(pos8 < ipIndex); LZ4MID_addPosition(hash8Table, h8, ipIndex); - if (ipIndex - pos8 <= LZ4_DISTANCE_MAX) { - /* match candidate found */ - if (pos8 >= prefixIdx) { - const BYTE* const matchPtr = prefixPtr + pos8 - prefixIdx; - assert(matchPtr < ip); - matchLength = LZ4_count(ip, matchPtr, matchlimit); - if (matchLength >= MINMATCH) { - DEBUGLOG(7, "found long match at pos %u (len=%u)", pos8, matchLength); - matchDistance = ipIndex - pos8; - goto _lz4mid_encode_sequence; - } - } else { - if (pos8 >= dictIdx) { - /* extDict match candidate */ - const BYTE* const matchPtr = dictStart + (pos8 - dictIdx); - const size_t safeLen = MIN(prefixIdx - pos8, (size_t)(matchlimit - ip)); - matchLength = LZ4_count(ip, matchPtr, ip + safeLen); - if (matchLength >= MINMATCH) { - DEBUGLOG(7, "found long match at ExtDict pos %u (len=%u)", pos8, matchLength); - matchDistance = ipIndex - pos8; - goto _lz4mid_encode_sequence; - } - } - } - } } + matchLength = LZ4MID_countMatch(ip, ipIndex, pos8, matchlimit, prefixPtr, prefixIdx, dictStart, dictIdx); + matchDistance = ipIndex - pos8; + } /* search short match */ - { U32 const h4 = LZ4MID_hash4Ptr(ip); + if (matchLength < MINMATCH) { + U32 const h4 = LZ4MID_hash4Ptr(ip); U32 const pos4 = hash4Table[h4]; assert(h4 < LZ4MID_HASHTABLESIZE); assert(pos4 < ipIndex); LZ4MID_addPosition(hash4Table, h4, ipIndex); - if (ipIndex - pos4 <= LZ4_DISTANCE_MAX) { - /* match candidate found */ - if (pos4 >= prefixIdx) { - /* only search within prefix */ - const BYTE* const matchPtr = prefixPtr + (pos4 - prefixIdx); - assert(matchPtr < ip); - assert(matchPtr >= prefixPtr); - matchLength = LZ4_count(ip, matchPtr, matchlimit); - if (matchLength >= MINMATCH) { - /* short match found, let's just check ip+1 for longer */ - U32 const h8 = LZ4MID_hash8Ptr(ip+1); - U32 const pos8 = hash8Table[h8]; - U32 const m2Distance = ipIndex + 1 - pos8; - matchDistance = ipIndex - pos4; - if ( m2Distance <= LZ4_DISTANCE_MAX - && pos8 >= prefixIdx /* only search within prefix */ - && likely(ip < mflimit) - ) { - const BYTE* const m2Ptr = prefixPtr + (pos8 - prefixIdx); - unsigned ml2 = LZ4_count(ip+1, m2Ptr, matchlimit); - if (ml2 > matchLength) { - LZ4MID_addPosition(hash8Table, h8, ipIndex+1); - ip++; - matchLength = ml2; - matchDistance = m2Distance; - } } - goto _lz4mid_encode_sequence; - } - } else { - if (pos4 >= dictIdx) { - /* extDict match candidate */ - const BYTE* const matchPtr = dictStart + (pos4 - dictIdx); - const size_t safeLen = MIN(prefixIdx - pos4, (size_t)(matchlimit - ip)); - matchLength = LZ4_count(ip, matchPtr, ip + safeLen); - if (matchLength >= MINMATCH) { - DEBUGLOG(7, "found match at ExtDict pos %u (len=%u)", pos4, matchLength); - matchDistance = ipIndex - pos4; - goto _lz4mid_encode_sequence; - } - } - } - } } - /* no match found in prefix */ + matchLength = LZ4MID_countMatch(ip, ipIndex, pos4, matchlimit, prefixPtr, prefixIdx, dictStart, dictIdx); + matchDistance = ipIndex - pos4; + } + if (matchLength >= MINMATCH) { + if ((matchLength < 8) && likely(ip < mflimit)) { + /* short match : check ip+1 for longer, except ip itself */ + U32 const h8 = LZ4MID_hash8Ptr(ip+1); + U32 const pos8 = hash8Table[h8]; + unsigned const ml2 = LZ4MID_countMatch(ip+1, ipIndex+1, pos8, matchlimit, prefixPtr, prefixIdx, dictStart, dictIdx); + if ((ml2 > matchLength) && (pos8 != ipIndex)) { + LZ4MID_addPosition(hash8Table, h8, ipIndex+1); + ip++; + matchLength = ml2; + matchDistance = ipIndex + 1 - pos8; + } } + goto _lz4mid_encode_sequence; + } + /* no match found in prefix nor extDict : search into the attached dictionary */ if ( (dict == usingDictCtxHc) && (ipIndex - gDictEndIndex < LZ4_DISTANCE_MAX - 8) ) { - /* search a match into external dictionary */ - LZ4HC_match_t dMatch = searchIntoDict(ip, ipIndex, + LZ4HC_match_t const dMatch = searchIntoDict(ip, ipIndex, matchlimit, ctx->dictCtx, gDictEndIndex); - if (dMatch.len >= MINMATCH) { - DEBUGLOG(7, "found Dictionary match (offset=%i)", dMatch.off); - assert(dMatch.back == 0); - matchLength = (unsigned)dMatch.len; - matchDistance = (unsigned)dMatch.off; - goto _lz4mid_encode_sequence; - } + assert(dMatch.back == 0); + matchLength = (unsigned)dMatch.len; + matchDistance = (unsigned)dMatch.off; + if (matchLength >= MINMATCH) goto _lz4mid_encode_sequence; } /* no match found */ ip += 1 + ((ip-anchor) >> 9); /* skip faster over incompressible data */ continue; _lz4mid_encode_sequence: + /* match found at ip : matchLength, matchDistance */ + DEBUGLOG(7, "found match at pos %u (len=%u, distance=%u)", + (U32)(ip - prefixPtr) + prefixIdx, matchLength, matchDistance); + /* catch back */ while (((ip > anchor) & ((U32)(ip-prefixPtr) > matchDistance)) && (unlikely(ip[-1] == ip[-(int)matchDistance-1]))) { ip--; matchLength++; }; - /* fill table with beginning of match */ - ADDPOS8(ip+1, ipIndex+1); - ADDPOS8(ip+2, ipIndex+2); - ADDPOS4(ip+1, ipIndex+1); + /* fill table with beginning of match + * note : ip may have moved since ipIndex (longer match at ip+1, catch back) */ + { U32 const startMatchIdx = (U32)(ip-prefixPtr) + prefixIdx; + ADDPOS8(ip+1, startMatchIdx+1); + ADDPOS8(ip+2, startMatchIdx+2); + ADDPOS4(ip+1, startMatchIdx+1); + } /* encode */ - { BYTE* const saved_op = op; - /* LZ4HC_encodeSequence always updates @op; on success, it updates @ip and @anchor */ - if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), - (int)matchLength, (int)matchDistance, - limit, oend) ) { - op = saved_op; /* restore @op value before failed LZ4HC_encodeSequence */ - goto _lz4mid_dest_overflow; - } + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + (int)matchLength, (int)matchDistance, + limit, oend) ) { + if (limit != fillOutput) return 0; /* compression failed */ + refEnd = ip + matchLength; + LZ4HC_encodeLastSequence_fillOutput(UPDATABLE(ip, op, anchor), + (int)matchLength, (int)matchDistance, oend); + break; } /* fill table with end of match */ @@ -708,34 +851,11 @@ _lz4mid_encode_sequence: } _lz4mid_last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ - size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; - size_t const totalSize = 1 + llAdd + lastRunSize; - if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ - if (limit && (op + totalSize > oend)) { - if (limit == limitedOutput) return 0; /* not enough space in @dst */ - /* adapt lastRunSize to fill 'dest' */ - lastRunSize = (size_t)(oend - op) - 1 /*token*/; - llAdd = (lastRunSize + 256 - RUN_MASK) / 256; - lastRunSize -= llAdd; - } - DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); - ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = (RUN_MASK << ML_BITS); - for(; accumulator >= 255 ; accumulator -= 255) - *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize << ML_BITS); - } - assert(lastRunSize <= (size_t)(oend - op)); - LZ4_memcpy(op, anchor, lastRunSize); - op += lastRunSize; - } + if (refEnd < ip) refEnd = ip; + if (LZ4HC_encodeLastLiterals(UPDATABLE(ip, op, anchor), iend, limit, oend)) + return 0; /* not enough space in @dst */ + if (ip < refEnd) /* fillOutput : the rest of input is left to the next block (input >= LZ4_minLength) */ + LZ4HC_dropPositions(ctx, ip, MIN(refEnd, iend - LZ4MID_HASHSIZE), lz4mid); /* End */ DEBUGLOG(5, "compressed %i bytes into %i bytes", *srcSizePtr, (int)((char*)op - dst)); @@ -743,36 +863,9 @@ _lz4mid_last_literals: assert(ip <= iend); *srcSizePtr = (int)(ip - (const BYTE*)src); assert((char*)op >= dst); - assert(op <= oend); + assert((char*)op - dst <= maxOutputSize); assert((char*)op - dst < INT_MAX); return (int)((char*)op - dst); - -_lz4mid_dest_overflow: - if (limit == fillOutput) { - /* Assumption : @ip, @anchor, @optr and @matchLength must be set correctly */ - size_t const ll = (size_t)(ip - anchor); - size_t const ll_addbytes = (ll + 240) / 255; - size_t const ll_totalCost = 1 + ll_addbytes + ll; - BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ - DEBUGLOG(6, "Last sequence is overflowing : %u literals, %u remaining space", - (unsigned)ll, (unsigned)(oend-op)); - if (op + ll_totalCost <= maxLitPos) { - /* ll validated; now adjust match length */ - size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); - size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); - assert(maxMlSize < INT_MAX); - if ((size_t)matchLength > maxMlSize) matchLength= (unsigned)maxMlSize; - if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + matchLength >= MFLIMIT) { - DEBUGLOG(6, "Let's encode a last sequence (ll=%u, ml=%u)", (unsigned)ll, matchLength); - LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), - (int)matchLength, (int)matchDistance, - notLimited, oend); - } } - DEBUGLOG(6, "Let's finish with a run of literals (%u bytes left)", (unsigned)(oend-op)); - goto _lz4mid_last_literals; - } - /* compression failed */ - return 0; } @@ -881,6 +974,72 @@ static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex) } typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; + +/* LZ4HC_sparsestChain() : + * A longer match must also match at every position of the current one (at @matchIndex, @longest bytes). + * Finds, among these positions, the chain jumping farthest back (sampled, with acceleration). + * @return : distance to the next candidate of that chain (1 if none is better), + * *matchChainPosPtr is set to its position within the match, if better */ +LZ4_FORCE_INLINE U32 +LZ4HC_sparsestChain(const U16* chainTable, U32 matchIndex, int longest, U32* matchChainPosPtr) +{ + int const kTrigger = 4; + U32 distanceToNextMatch = 1; + int const end = longest - MINMATCH + 1; + int step = 1; + int accel = 1 << kTrigger; + int pos; + for (pos = 0; pos < end; pos += step) { + U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos); + step = (accel++ >> kTrigger); + if (candidateDist > distanceToNextMatch) { + distanceToNextMatch = candidateDist; + *matchChainPosPtr = (U32)pos; + accel = 1 << kTrigger; + } } + return distanceToNextMatch; +} + +/* LZ4HC_patternSegment() : + * Measures the run of @pattern (a repeated byte, or 2 alternating bytes) around @candidateIdx, + * across the external dictionary / prefix boundary, without going below @lowestMatchIndex. + * @return : 0 if @pattern isn't present at @candidateIdx, + * 1 otherwise, with *forwardPtr bytes from @candidateIdx (included) and *backPtr bytes before it */ +LZ4_FORCE_INLINE int +LZ4HC_patternSegment(const LZ4HC_CCtx_internal* hc4, U32 candidateIdx, U32 pattern, + const BYTE* iHighLimit, U32 lowestMatchIndex, + size_t* forwardPtr, size_t* backPtr) +{ + const BYTE* const prefixPtr = hc4->prefixStart; + U32 const prefixIdx = hc4->dictLimit; + const BYTE* const dictStart = hc4->dictStart; + U32 const dictIdx = hc4->lowLimit; + const BYTE* const dictEnd = dictStart + prefixIdx - dictIdx; + int const extDict = candidateIdx < prefixIdx; + const BYTE* const matchPtr = extDict ? dictStart + (candidateIdx - dictIdx) : prefixPtr + (candidateIdx - prefixIdx); + size_t forward, back; + if (LZ4_read32(matchPtr) != pattern) return 0; + + { const BYTE* const iLimit = extDict ? dictEnd : iHighLimit; + forward = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern); + if (extDict && matchPtr + forward == iLimit) { /* continues into the prefix */ + U32 const rotatedPattern = LZ4HC_rotatePattern(forward, pattern); + forward += LZ4HC_countPattern(prefixPtr, iHighLimit, rotatedPattern); + } } + + back = LZ4HC_reverseCountPattern(matchPtr, extDict ? dictStart : prefixPtr, pattern); + if (!extDict && matchPtr - back == prefixPtr && dictIdx < prefixIdx) { /* continues into the external dictionary */ + U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)back), pattern); + back += LZ4HC_reverseCountPattern(dictEnd, dictStart, rotatedPattern); + } + /* don't go below lowestMatchIndex */ + back = candidateIdx - MAX(candidateIdx - (U32)back, lowestMatchIndex); + assert(candidateIdx - back >= lowestMatchIndex); + + *forwardPtr = forward; + *backPtr = back; + return 1; +} typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; @@ -905,8 +1064,8 @@ LZ4HC_InsertAndGetWiderMatch ( const U32 lowestMatchIndex = (withinStartDistance) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; const BYTE* const dictStart = hc4->dictStart; const U32 dictIdx = hc4->lowLimit; - const BYTE* const dictEnd = dictStart + prefixIdx - dictIdx; int const lookBackLength = (int)(ip-iLowLimit); + int const maxLength = (int)(iHighLimit-iLowLimit); /* no match can be longer */ int nbAttempts = maxNbAttempts; U32 matchChainPos = 0; U32 const pattern = LZ4_read32(ip); @@ -922,7 +1081,9 @@ LZ4HC_InsertAndGetWiderMatch ( DEBUGLOG(7, "First candidate match for pos %u found at index %u / %u (lowestMatchIndex)", ipIndex, matchIndex, lowestMatchIndex); - while ((matchIndex>=lowestMatchIndex) && (nbAttempts>0)) { + /* Candidates only replace the current match if strictly longer, + * so stopping once @longest reaches @maxLength doesn't change the result. */ + while ((matchIndex>=lowestMatchIndex) && (nbAttempts>0) && (longest select a better chain */ + if (chainSwap && matchLength==longest && matchIndex + (U32)longest <= ipIndex) { /* better match => select a better chain */ + U32 const distanceToNextMatch = LZ4HC_sparsestChain(chainTable, matchIndex, longest, &matchChainPos); assert(lookBackLength==0); /* search forward only */ - if (matchIndex + (U32)longest <= ipIndex) { - int const kTrigger = 4; - U32 distanceToNextMatch = 1; - int const end = longest - MINMATCH + 1; - int step = 1; - int accel = 1 << kTrigger; - int pos; - for (pos = 0; pos < end; pos += step) { - U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos); - step = (accel++ >> kTrigger); - if (candidateDist > distanceToNextMatch) { - distanceToNextMatch = candidateDist; - matchChainPos = (U32)pos; - accel = 1 << kTrigger; - } } - if (distanceToNextMatch > 1) { - if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ - matchIndex -= distanceToNextMatch; - continue; - } } } + if (distanceToNextMatch > 1) { + if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ + matchIndex -= distanceToNextMatch; + continue; + } } { U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex); if (patternAnalysis && distNextMatch==1 && matchChainPos==0) { U32 const matchCandidateIdx = matchIndex-1; + size_t forwardPatternLength, backLength; /* may be a repeated pattern */ if (repeat == rep_untested) { if ( ((pattern & 0xFFFF) == (pattern >> 16)) @@ -1001,64 +1151,42 @@ LZ4HC_InsertAndGetWiderMatch ( repeat = rep_not; } } if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex) - && LZ4HC_protectDictEnd(prefixIdx, matchCandidateIdx) ) { - const int extDict = matchCandidateIdx < prefixIdx; - const BYTE* const matchPtr = extDict ? dictStart + (matchCandidateIdx - dictIdx) : prefixPtr + (matchCandidateIdx - prefixIdx); - if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ - const BYTE* const iLimit = extDict ? dictEnd : iHighLimit; - size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern); - if (extDict && matchPtr + forwardPatternLength == iLimit) { - U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern); - forwardPatternLength += LZ4HC_countPattern(prefixPtr, iHighLimit, rotatedPattern); - } - { const BYTE* const lowestMatchPtr = extDict ? dictStart : prefixPtr; - size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); - size_t currentSegmentLength; - if (!extDict - && matchPtr - backLength == prefixPtr - && dictIdx < prefixIdx) { - U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern); - backLength += LZ4HC_reverseCountPattern(dictEnd, dictStart, rotatedPattern); - } - /* Limit backLength not go further than lowestMatchIndex */ - backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex); - assert(matchCandidateIdx - backLength >= lowestMatchIndex); - currentSegmentLength = backLength + forwardPatternLength; - /* Adjust to end of pattern if the source pattern fits, otherwise the beginning of the pattern */ - if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ - && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ - U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ - if (LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) - matchIndex = newMatchIndex; - else { - /* Can only happen if started in the prefix */ - assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict); - matchIndex = prefixIdx; - } - } else { - U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ - if (!LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) { - assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict); - matchIndex = prefixIdx; - } else { - matchIndex = newMatchIndex; - if (lookBackLength==0) { /* no back possible */ - size_t const maxML = MIN(currentSegmentLength, srcPatternLength); - if ((size_t)longest < maxML) { - assert(prefixPtr - prefixIdx + matchIndex != ip); - if ((size_t)(ip - prefixPtr) + prefixIdx - matchIndex > LZ4_DISTANCE_MAX) break; - assert(maxML < 2 GB); - longest = (int)maxML; - offset = (int)(ipIndex - matchIndex); - assert(sBack == 0); - DEBUGLOG(7, "Found repeat pattern match of len=%i, offset=%i", longest, offset); - } - { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); - if (distToNextPattern > matchIndex) break; /* avoid overflow */ - matchIndex -= distToNextPattern; - } } } } } + && LZ4HC_protectDictEnd(prefixIdx, matchCandidateIdx) + && LZ4HC_patternSegment(hc4, matchCandidateIdx, pattern, iHighLimit, lowestMatchIndex, + &forwardPatternLength, &backLength) ) { + size_t const currentSegmentLength = backLength + forwardPatternLength; + /* if the full source pattern fits (and hasn't been reached yet), jump to its best position : + * at the end of the segment, it might be followed by more match. + * Otherwise, jump to the farthest position in the segment : the longest match, plus maybe some back */ + int const fullPatternFits = (currentSegmentLength >= srcPatternLength) + && (forwardPatternLength <= srcPatternLength); + U32 const newMatchIndex = fullPatternFits ? + matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength : + matchCandidateIdx - (U32)backLength; + if (!LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) { + /* Can only happen if started in the prefix */ + assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && matchCandidateIdx >= prefixIdx); + matchIndex = prefixIdx; continue; - } } + } + matchIndex = newMatchIndex; + if (!fullPatternFits && lookBackLength==0) { /* no back possible : the match length is known */ + size_t const maxML = MIN(currentSegmentLength, srcPatternLength); + if ((size_t)longest < maxML) { + assert(prefixPtr - prefixIdx + matchIndex != ip); + if ((size_t)(ip - prefixPtr) + prefixIdx - matchIndex > LZ4_DISTANCE_MAX) break; + assert(maxML < 2 GB); + longest = (int)maxML; + offset = (int)(ipIndex - matchIndex); + assert(sBack == 0); + DEBUGLOG(7, "Found repeat pattern match of len=%i, offset=%i", longest, offset); + } + { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); + if (distToNextPattern > matchIndex) break; /* avoid overflow */ + matchIndex -= distToNextPattern; + } } + continue; + } } } /* PA optimization */ /* follow current chain */ @@ -1068,13 +1196,14 @@ LZ4HC_InsertAndGetWiderMatch ( if ( dict == usingDictCtxHc && nbAttempts > 0 + && longest < maxLength && withinStartDistance) { size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->prefixStart) + dictCtx->dictLimit; U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; assert(dictEndOffset <= 1 GB); matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; if (dictMatchIndex>0) DEBUGLOG(7, "dictEndOffset = %zu, dictMatchIndex = %u => relative matchIndex = %i", dictEndOffset, dictMatchIndex, (int)dictMatchIndex - (int)dictEndOffset); - while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) { + while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && longest < maxLength && nbAttempts--) { const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + dictMatchIndex; if (LZ4_read32(matchPtr) == pattern) { @@ -1121,10 +1250,16 @@ LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table wi } +/* preconditions: + * - *srcSizePtr within [1, LZ4_MAX_INPUT_SIZE] + * - src is valid + * - maxOutputSize >= 1 + * - dst is valid + */ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( LZ4HC_CCtx_internal* const ctx, - const char* const source, - char* const dest, + const char* const src, + char* const dst, int* srcSizePtr, int const maxOutputSize, int maxNbAttempts, @@ -1135,15 +1270,15 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( const int inputSize = *srcSizePtr; const int patternAnalysis = (maxNbAttempts > 128); /* levels 9+ */ - const BYTE* ip = (const BYTE*) source; + const BYTE* ip = (const BYTE*)src; const BYTE* anchor = ip; const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); + const BYTE* mflimit; /* set after the size check : would precede a small input */ + const BYTE* matchlimit; - BYTE* optr = (BYTE*) dest; - BYTE* op = (BYTE*) dest; - BYTE* oend = op + maxOutputSize; + BYTE* op = (BYTE*) dst; + /* fillOutput : keep room for the final run of literals (LZ4 format restriction) */ + BYTE* const oend = op + maxOutputSize - ((limit == fillOutput) ? LASTLITERALS : 0); const BYTE* start0; const BYTE* start2 = NULL; @@ -1153,9 +1288,17 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( /* init */ DEBUGLOG(5, "LZ4HC_compress_hashChain (dict?=>%i)", dict); + + /* preconditions verifications */ + assert(*srcSizePtr >= 1); + assert(src != NULL); + assert(maxOutputSize >= 1); + assert(dst != NULL); + *srcSizePtr = 0; - if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + mflimit = iend - MFLIMIT; + matchlimit = (iend - LASTLITERALS); /* Main Loop */ while (ip <= mflimit) { @@ -1166,6 +1309,8 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( start0 = ip; m0 = m1; _Search2: + /* m1 (at ip) is the next match to encode, m0 (at start0 <= ip) its fallback. + * Search a longer match m2, starting before the end of m1. */ DEBUGLOG(7, "_Search2 (currently found match of size %i)", m1.len); if (ip+m1.len <= mflimit) { start2 = ip + m1.len - 2; @@ -1177,8 +1322,7 @@ _Search2: m2 = nomatch; /* do not search further */ } - if (m2.len <= m1.len) { /* No better match => encode ML1 immediately */ - optr = op; + if (m2.len <= m1.len) { /* No better match => encode m1 immediately */ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, m1.off, limit, oend) ) @@ -1186,194 +1330,122 @@ _Search2: continue; } - if (start0 < ip) { /* first match was skipped at least once */ - if (start2 < ip + m0.len) { /* squeezing ML1 between ML0(original ML1) and ML2 */ - ip = start0; m1 = m0; /* restore initial Match1 */ - } } + if ((start0 < ip) && (start2 < ip + m0.len)) { + /* m1 was skipped at least once, and m2 starts early enough : restore m0 */ + ip = start0; m1 = m0; + } - /* Here, start0==ip */ - if ((start2 - ip) < 3) { /* First Match too small : removed */ + if ((start2 - ip) < 3) { /* m1 too short before m2 : removed */ ip = start2; m1 = m2; goto _Search2; } -_Search3: - if ((start2 - ip) < OPTIMAL_ML) { - int correction; - int new_ml = m1.len; - if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; - if (ip+new_ml > start2 + m2.len - MINMATCH) - new_ml = (int)(start2 - ip) + m2.len - MINMATCH; - correction = new_ml - (int)(start2 - ip); - if (correction > 0) { - start2 += correction; - m2.len -= correction; + /* m2 (at start2) is the candidate after m1. + * Search a longer match m3, starting before the end of m2. */ + while (1) { + /* m1 will be cut where m2 starts, but keeps up to OPTIMAL_ML bytes (no extra length byte) : + * m2 is shifted after them, as long as it keeps at least MINMATCH bytes. */ + if ((start2 - ip) < OPTIMAL_ML) { + int const gap = (int)(start2 - ip); + int newML = MIN(m1.len, OPTIMAL_ML); + if (newML > gap + m2.len - MINMATCH) + newML = gap + m2.len - MINMATCH; + if (newML > gap) { /* shift m2 */ + start2 += newML - gap; + m2.len -= newML - gap; + } } - } - if (start2 + m2.len <= mflimit) { - start3 = start2 + m2.len - 3; - m3 = LZ4HC_InsertAndGetWiderMatch(ctx, - start3, start2, matchlimit, m2.len, - maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio); - start3 += m3.back; - } else { - m3 = nomatch; /* do not search further */ - } + if (start2 + m2.len <= mflimit) { + start3 = start2 + m2.len - 3; + m3 = LZ4HC_InsertAndGetWiderMatch(ctx, + start3, start2, matchlimit, m2.len, + maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio); + start3 += m3.back; + } else { + m3 = nomatch; /* do not search further */ + } - if (m3.len <= m2.len) { /* No better match => encode ML1 and ML2 */ - /* ip & ref are known; Now for ml */ + if ((m3.len > m2.len) && (start3 < ip+m1.len+3)) { /* not enough room for m2 between m1 and m3 */ + if (start3 >= (ip+m1.len)) { + /* m3 starts after m1 : encode m1, then m3 becomes m1, + * and m2, moved after m1 if needed (replaced by m3 if too short), becomes the fallback m0 */ + if (start2 < ip+m1.len) { + int correction = (int)(ip+m1.len - start2); + start2 += correction; + m2.len -= correction; + if (m2.len < MINMATCH) { + start2 = start3; + m2 = m3; + } + } + + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + m1.len, m1.off, + limit, oend) ) + goto _dest_overflow; + ip = start3; + m1 = m3; + + start0 = start2; + m0 = m2; + goto _Search2; + } + + /* m3 starts within m1 : m3 replaces m2 */ + start2 = start3; + m2 = m3; + continue; + } + + /* encode m1, cut where m2 starts */ if (start2 < ip+m1.len) m1.len = (int)(start2 - ip); - /* Now, encode 2 sequences */ - optr = op; if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, m1.off, limit, oend) ) goto _dest_overflow; - ip = start2; - optr = op; - if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), - m2.len, m2.off, - limit, oend) ) { - m1 = m2; - goto _dest_overflow; - } - continue; - } - if (start3 < ip+m1.len+3) { /* Not enough space for match 2 : remove it */ - if (start3 >= (ip+m1.len)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ - if (start2 < ip+m1.len) { - int correction = (int)(ip+m1.len - start2); - start2 += correction; - m2.len -= correction; - if (m2.len < MINMATCH) { - start2 = start3; - m2 = m3; - } - } + /* m2 becomes m1 */ + ip = start2; m1 = m2; - optr = op; + if (m3.len <= m1.len) { /* No better match => encode m1 (former m2) */ if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, m1.off, limit, oend) ) goto _dest_overflow; - ip = start3; - m1 = m3; - - start0 = start2; - m0 = m2; - goto _Search2; + break; } - start2 = start3; - m2 = m3; - goto _Search3; + /* m3 becomes m2 */ + start2 = start3; m2 = m3; } - - /* - * OK, now we have 3 ascending matches; - * let's write the first one ML1. - * ip & ref are known; Now decide ml. - */ - if (start2 < ip+m1.len) { - if ((start2 - ip) < OPTIMAL_ML) { - int correction; - if (m1.len > OPTIMAL_ML) m1.len = OPTIMAL_ML; - if (ip + m1.len > start2 + m2.len - MINMATCH) - m1.len = (int)(start2 - ip) + m2.len - MINMATCH; - correction = m1.len - (int)(start2 - ip); - if (correction > 0) { - start2 += correction; - m2.len -= correction; - } - } else { - m1.len = (int)(start2 - ip); - } - } - optr = op; - if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), - m1.len, m1.off, - limit, oend) ) - goto _dest_overflow; - - /* ML2 becomes ML1 */ - ip = start2; m1 = m2; - - /* ML3 becomes ML2 */ - start2 = start3; m2 = m3; - - /* let's find a new ML3 */ - goto _Search3; } _last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ - size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; - size_t const totalSize = 1 + llAdd + lastRunSize; - if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ - if (limit && (op + totalSize > oend)) { - if (limit == limitedOutput) return 0; - /* adapt lastRunSize to fill 'dest' */ - lastRunSize = (size_t)(oend - op) - 1 /*token*/; - llAdd = (lastRunSize + 256 - RUN_MASK) / 256; - lastRunSize -= llAdd; - } - DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); - ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = (RUN_MASK << ML_BITS); - for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize << ML_BITS); - } - LZ4_memcpy(op, anchor, lastRunSize); - op += lastRunSize; - } + if (LZ4HC_encodeLastLiterals(UPDATABLE(ip, op, anchor), iend, limit, oend)) + return 0; /* not enough space in @dst */ /* End */ - *srcSizePtr = (int) (((const char*)ip) - source); - return (int) (((char*)op)-dest); + *srcSizePtr = (int) (((const char*)ip) - src); + return (int) (((char*)op)-dst); _dest_overflow: - if (limit == fillOutput) { - /* Assumption : @ip, @anchor, @optr and @m1 must be set correctly */ - size_t const ll = (size_t)(ip - anchor); - size_t const ll_addbytes = (ll + 240) / 255; - size_t const ll_totalCost = 1 + ll_addbytes + ll; - BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ - DEBUGLOG(6, "Last sequence overflowing"); - op = optr; /* restore correct out pointer */ - if (op + ll_totalCost <= maxLitPos) { - /* ll validated; now adjust match length */ - size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); - size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); - assert(maxMlSize < INT_MAX); assert(m1.len >= 0); - if ((size_t)m1.len > maxMlSize) m1.len = (int)maxMlSize; - if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + m1.len >= MFLIMIT) { - LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, m1.off, notLimited, oend); - } } - goto _last_literals; - } - /* compression failed */ - return 0; + if (limit != fillOutput) return 0; /* compression failed */ + LZ4HC_encodeLastSequence_fillOutput(UPDATABLE(ip, op, anchor), m1.len, m1.off, oend); + goto _last_literals; } static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx, const char* const source, char* dst, int* srcSizePtr, int dstCapacity, - int const nbSearches, size_t sufficient_len, - const limitedOutput_directive limit, int const fullUpdate, + int const nbSearches, int const nbSearchesSeq, int const targetLength, + const limitedOutput_directive limit, const dictCtx_directive dict, const HCfavor_e favorDecSpeed); -LZ4_FORCE_INLINE int +static int LZ4HC_compress_generic_internal ( LZ4HC_CCtx_internal* const ctx, const char* const src, @@ -1385,15 +1457,20 @@ LZ4HC_compress_generic_internal ( const dictCtx_directive dict ) { - DEBUGLOG(5, "LZ4HC_compress_generic_internal(src=%p, srcSize=%d)", - src, *srcSizePtr); + DEBUGLOG(5, "LZ4HC_compress_generic_internal(src=%p, srcSize=%d, dstCapacity=%d)", + src, *srcSizePtr, dstCapacity); - if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */ + /* input sanitization */ if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ + if (dstCapacity < 1) return 0; /* Invalid: impossible to store anything */ + assert(dst); /* since dstCapacity >= 1, dst must be valid */ + if (*srcSizePtr == 0) { *dst = 0; return 1; } + assert(src != NULL); /* since *srcSizePtr >= 1, src must be valid */ ctx->end += *srcSizePtr; { cParams_t const cParam = LZ4HC_getCLevelParams(cLevel); HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio; + int const srcSize = *srcSizePtr; int result; if (cParam.strat == lz4mid) { @@ -1408,11 +1485,23 @@ LZ4HC_compress_generic_internal ( assert(cParam.strat == lz4opt); result = LZ4HC_compress_optimal(ctx, src, dst, srcSizePtr, dstCapacity, - cParam.nbSearches, cParam.targetLength, limit, - cLevel >= LZ4HC_CLEVEL_MAX, /* ultra mode */ + cParam.nbSearches, cParam.nbSearchesSeq, (int)cParam.targetLength, limit, dict, favor); } - if (result <= 0) ctx->dirty = 1; + if ((limit == fillOutput) && (result > 0) && (*srcSizePtr < srcSize)) { + /* fillOutput : compression stopped before the end of input. + * The rest of input is left to the next block : it isn't history yet, + * and the positions already inserted into it are dropped (lz4mid drops its own). + * Note : a failed compression is different : lz4frame stores its block uncompressed, + * and the next block still references it. */ + const BYTE* const consumedEnd = (const BYTE*)src + *srcSizePtr; + U32 const consumedIdx = ctx->dictLimit + (U32)(consumedEnd - ctx->prefixStart); + if ((cParam.strat != lz4mid) && (ctx->nextToUpdate > consumedIdx)) { + LZ4HC_dropPositions(ctx, consumedEnd, consumedEnd + (ctx->nextToUpdate - consumedIdx), cParam.strat); + ctx->nextToUpdate = consumedIdx; + } + ctx->end = consumedEnd; + } return result; } } @@ -1457,10 +1546,15 @@ LZ4HC_compress_generic_dictCtx ( if (position >= 64 KB) { ctx->dictCtx = NULL; return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); - } else if (position == 0 && *srcSizePtr > 4 KB && isStateCompatible(ctx, ctx->dictCtx)) { + } else if (position == 0 && *srcSizePtr >= LZ4HC_getCLevelParams(cLevel).dictCopyMin && isStateCompatible(ctx, ctx->dictCtx)) { + /* parameters and extension belong to the working context, not to the dictionary */ + LZ4_i8 const favorDecSpeed = ctx->favorDecSpeed; + LZ4_i8 const hasWorkspace = ctx->hasWorkspace; LZ4_memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); LZ4HC_setExternalDict(ctx, (const BYTE *)src); ctx->compressionLevel = (short)cLevel; + ctx->favorDecSpeed = favorDecSpeed; + ctx->hasWorkspace = hasWorkspace; return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); } else { return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtxHc); @@ -1522,17 +1616,18 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { int cSize; -#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 - LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); - if (statePtr==NULL) return 0; -#else - LZ4_streamHC_t state; - LZ4_streamHC_t* const statePtr = &state; -#endif DEBUGLOG(5, "LZ4_compress_HC") - cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 - FREEMEM(statePtr); + { /* extended state : a single allocation, including the temporary buffer of levels >= 10 */ + LZ4_streamHC_t* const statePtr = LZ4_initStreamHC(ALLOC(LZ4_STREAMHC_EXTSIZE), LZ4_STREAMHC_EXTSIZE); + if (statePtr==NULL) return 0; + cSize = LZ4_compress_HC_extStateHC_fastReset(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); + FREEMEM(statePtr); + } +#else + { LZ4_streamHC_t state; + cSize = LZ4_compress_HC_extStateHC(&state, src, dst, srcSize, dstCapacity, compressionLevel); + } #endif return cSize; } @@ -1557,8 +1652,9 @@ int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* s LZ4_streamHC_t* LZ4_createStreamHC(void) { LZ4_streamHC_t* const state = - (LZ4_streamHC_t*)ALLOC_AND_ZERO(sizeof(LZ4_streamHC_t)); + (LZ4_streamHC_t*)ALLOC_AND_ZERO(LZ4_STREAMHC_EXTSIZE); /* zeroed : initialized */ if (state == NULL) return NULL; + state->internal_donotuse.hasWorkspace = 1; LZ4_setCompressionLevel(state, LZ4HC_CLEVEL_DEFAULT); return state; } @@ -1583,15 +1679,22 @@ LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size) if (!LZ4_isAligned(buffer, LZ4_streamHC_t_alignment())) return NULL; /* init */ { LZ4HC_CCtx_internal* const hcstate = &(LZ4_streamHCPtr->internal_donotuse); - MEM_INIT(hcstate, 0, sizeof(*hcstate)); } + MEM_INIT(hcstate, 0, sizeof(*hcstate)); + hcstate->hasWorkspace = (size >= LZ4_STREAMHC_EXTSIZE); } LZ4_setCompressionLevel(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT); return LZ4_streamHCPtr; } +/* size of an initialized state, to initialize it again : extended states are followed by their buffer */ +static size_t LZ4HC_sizeofState(const LZ4_streamHC_t* s) +{ + return s->internal_donotuse.hasWorkspace ? LZ4_STREAMHC_EXTSIZE : sizeof(*s); +} + /* just a stub */ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { - LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + LZ4_initStreamHC(LZ4_streamHCPtr, LZ4HC_sizeofState(LZ4_streamHCPtr)); LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); } @@ -1599,15 +1702,12 @@ void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev { LZ4HC_CCtx_internal* const s = &LZ4_streamHCPtr->internal_donotuse; DEBUGLOG(5, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); - if (s->dirty) { - LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); - } else { - assert(s->end >= s->prefixStart); - s->dictLimit += (U32)(s->end - s->prefixStart); - s->prefixStart = NULL; - s->end = NULL; - s->dictCtx = NULL; - } + /* also after a failed compression : its positions are below @end, hence out of reach of the next stream */ + assert(s->end >= s->prefixStart); + s->dictLimit += (U32)(s->end - s->prefixStart); + s->prefixStart = NULL; + s->end = NULL; + s->dictCtx = NULL; LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); } @@ -1632,16 +1732,19 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; cParams_t cp; DEBUGLOG(4, "LZ4_loadDictHC(ctx:%p, dict:%p, dictSize:%d, clevel=%d)", LZ4_streamHCPtr, dictionary, dictSize, ctxPtr->compressionLevel); - assert(dictSize >= 0); assert(LZ4_streamHCPtr != NULL); + if (dictSize < 0) dictSize = 0; /* invalid size : no dictionary */ if (dictSize > 64 KB) { dictionary += (size_t)dictSize - 64 KB; dictSize = 64 KB; } - /* need a full initialization, there are bad side-effects when using resetFast() */ + /* need a full initialization, there are bad side-effects when using resetFast(), + * but preserve parameters */ { int const cLevel = ctxPtr->compressionLevel; - LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + LZ4_i8 const favorDecSpeed = ctxPtr->favorDecSpeed; + LZ4_initStreamHC(LZ4_streamHCPtr, LZ4HC_sizeofState(LZ4_streamHCPtr)); LZ4_setCompressionLevel(LZ4_streamHCPtr, cLevel); + ctxPtr->favorDecSpeed = favorDecSpeed; cp = LZ4HC_getCLevelParams(cLevel); } LZ4HC_init_internal (ctxPtr, (const BYTE*)dictionary); @@ -1663,9 +1766,14 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock); - if ( (ctxPtr->end >= ctxPtr->prefixStart + 4) - && (LZ4HC_getCLevelParams(ctxPtr->compressionLevel).strat != lz4mid) ) { - LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + if (LZ4HC_getCLevelParams(ctxPtr->compressionLevel).strat != lz4mid) { + U32 const endIdx = ctxPtr->dictLimit + (U32)(ctxPtr->end - ctxPtr->prefixStart); + U32 idx; + if (ctxPtr->end >= ctxPtr->prefixStart + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + /* last positions can't be inserted : don't leave stale chains from previous history */ + for (idx = ctxPtr->nextToUpdate; idx < endIdx; idx++) + DELTANEXTU16(ctxPtr->chainTable, idx) = 0; } /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ @@ -1747,11 +1855,14 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; int const prefixSize = (int)(streamPtr->end - streamPtr->prefixStart); DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); + /* nothing compressed yet (new or reset stream) : nothing to save, + * the stream remains to be initialized by next compression (attached dictionary included) */ + if (streamPtr->prefixStart == NULL) return 0; assert(prefixSize >= 0); if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 4) dictSize = 0; if (dictSize > prefixSize) dictSize = prefixSize; - if (safeBuffer == NULL) assert(dictSize == 0); + if (safeBuffer == NULL) assert(dictSize == 0); /* a NULL buffer with !0 size is invalid */ if (dictSize > 0) LZ4_memmove(safeBuffer, streamPtr->end - dictSize, (size_t)dictSize); { U32 const endIndex = (U32)(streamPtr->end - streamPtr->prefixStart) + streamPtr->dictLimit; @@ -1773,18 +1884,32 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS typedef struct { int price; int off; - int mlen; + int mlen; /* 1 means literal */ int litlen; } LZ4HC_optimal_t; -/* price in bytes */ +#define TRAILING_LITERALS 3 + +/* Prices are in 1/LZ4HC_PRICE_SCALE byte, + * leaving room for a price per sequence, lower than a byte. */ +#define LZ4HC_PRICE_SCALE 4 + +/* favorDecSpeed : each sequence costs decoding time, so it also costs LZ4HC_SEQ_PRICE_FDS. + * Removing a sequence almost always costs at least one byte, + * so this price, lower than a byte, mostly acts as a tie-breaker : + * among solutions of the same size, the one with fewer sequences is preferred. + * They are found at positions only worth searching to save a sequence, + * which are searched if at most LZ4HC_SEQ_SEARCH_DIST before the end of the furthest match. */ +#define LZ4HC_SEQ_PRICE_FDS 1 +#define LZ4HC_SEQ_SEARCH_DIST 4 + LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) { int price = litlen; assert(litlen >= 0); if (litlen >= (int)RUN_MASK) price += 1 + ((litlen-(int)RUN_MASK) / 255); - return price; + return price * LZ4HC_PRICE_SCALE; } /* requires mlen >= MINMATCH */ @@ -1794,34 +1919,261 @@ LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) assert(litlen >= 0); assert(mlen >= MINMATCH); - price += LZ4HC_literalsPrice(litlen); - if (mlen >= (int)(ML_MASK+MINMATCH)) price += 1 + ((mlen-(int)(ML_MASK+MINMATCH)) / 255); - return price; + return price * LZ4HC_PRICE_SCALE + LZ4HC_literalsPrice(litlen); +} + +/* LZ4HC_pricedLitlen() : + * All solutions of a window start with the same run of @llen literals, + * so only price differences matter, and they depend on @llen + * only through its position relative to the 255-steps of the literal length field. + * Reduce it accordingly : scaled prices of long runs would overflow. */ +LZ4_FORCE_INLINE int LZ4HC_pricedLitlen(int const llen) +{ + int const reduceFrom = (int)RUN_MASK + 255; + assert(llen >= 0); + if (llen < reduceFrom) return llen; + return reduceFrom + ((llen - (int)RUN_MASK) % 255); } LZ4_FORCE_INLINE LZ4HC_match_t -LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, - const BYTE* ip, const BYTE* const iHighLimit, - int minLen, int nbSearches, - const dictCtx_directive dict, - const HCfavor_e favorDecSpeed) +LZ4HC_FindLongestMatch(LZ4HC_CCtx_internal* const ctx, + const BYTE* ip, const BYTE* const iHighLimit, + int nbSearches, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) { LZ4HC_match_t const match0 = { 0 , 0, 0 }; - /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), - * but this won't be the case here, as we define iLowLimit==ip, - ** so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ - LZ4HC_match_t md = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed); + /* note : LZ4HC_InsertAndGetWiderMatch() can extend a match backward (md.back < 0), + * but not here, since iLowLimit == ip */ + LZ4HC_match_t md = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, MINMATCH-1, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed); assert(md.back == 0); - if (md.len <= minLen) return match0; + if (md.len < MINMATCH) return match0; if (favorDecSpeed) { if ((md.len>18) & (md.len<=36)) md.len=18; /* favor dec.speed (shortcut) */ } return md; } +LZ4_FORCE_INLINE void +LZ4HC_setLiteral(LZ4HC_optimal_t* opt, int pos, int litlen, int price) +{ + opt[pos].mlen = 1; + opt[pos].off = 0; + opt[pos].litlen = litlen; + opt[pos].price = price; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", pos, price, litlen); +} + +LZ4_FORCE_INLINE void +LZ4HC_setMatch(LZ4HC_optimal_t* opt, int pos, int mlen, int off, int price) +{ + opt[pos].mlen = mlen; + opt[pos].off = off; + opt[pos].litlen = 0; + opt[pos].price = price; + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)", pos, price, mlen); +} + +/* price at the start of the literal run ending at @pos + * (0 if this run started before position 0, i.e. it includes literals since anchor) */ +LZ4_FORCE_INLINE int LZ4HC_runStartPrice(const LZ4HC_optimal_t* opt, int pos) +{ + int const ll = opt[pos].litlen; + return (ll <= pos) ? opt[pos - ll].price : 0; +} + +/* LZ4HC_addMatch() : + * Update prices of positions reachable from @cur using @match. + * Positions beyond @lastPos + TRAILING_LITERALS are not reached yet. + * @return : updated @lastPos (end of the furthest match) */ +LZ4_FORCE_INLINE int +LZ4HC_addMatch(LZ4HC_optimal_t* opt, int cur, LZ4HC_match_t match, + int lastPos, const HCfavor_e favorDecSpeed) +{ + int const ll = opt[cur].litlen; + int const basePrice = LZ4HC_runStartPrice(opt, cur) + (favorDecSpeed ? LZ4HC_SEQ_PRICE_FDS : 0); + int ml; + assert(cur + match.len < LZ4_OPT_NUM); + assert((U32)favorDecSpeed <= 1); + for (ml = MINMATCH; ml <= match.len; ml++) { + int const pos = cur + ml; + int const price = basePrice + LZ4HC_sequencePrice(ll, ml); + /* on equal price, favorDecSpeed keeps the existing solution */ + if ( pos > lastPos + TRAILING_LITERALS + || price <= opt[pos].price - (int)favorDecSpeed) { + LZ4HC_setMatch(opt, pos, ml, match.off, price); + if ((ml == match.len) && (lastPos < pos)) lastPos = pos; + } } + + /* complete following positions with literals */ + { int addLit; + assert(opt[lastPos].mlen >= MINMATCH); + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit++) + LZ4HC_setLiteral(opt, lastPos + addLit, addLit, + opt[lastPos].price + LZ4HC_literalsPrice(addLit)); + } + return lastPos; +} + + +/* preconditions: + * - *srcSizePtr within [1, LZ4_MAX_INPUT_SIZE] + * - src is valid + * - maxOutputSize >= 1 + * - dst is valid + * - @opt : table of LZ4_OPT_NUM + TRAILING_LITERALS cells + */ +LZ4_FORCE_INLINE int +LZ4HC_compress_optimal_internal(LZ4HC_CCtx_internal* ctx, + const char* const source, + char* dst, + int* srcSizePtr, + int dstCapacity, + int const nbSearches, + int const nbSearchesSeq, + int const targetLength, + const limitedOutput_directive limit, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed, + LZ4HC_optimal_t* const opt) +{ + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const int inputSize = *srcSizePtr; + const BYTE* const iend = ip + inputSize; + const BYTE* mflimit; /* set after the size check : would precede a small input */ + const BYTE* matchlimit; + BYTE* op = (BYTE*) dst; + /* fillOutput : keep room for the final run of literals (LZ4 format restriction) */ + BYTE* const oend = op + dstCapacity - ((limit == fillOutput) ? LASTLITERALS : 0); + int const sufficientLen = MIN(targetLength, LZ4_OPT_NUM-1); + + DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity); + + /* preconditions verifications */ + assert(dstCapacity > 0); + assert(dst != NULL); + assert(*srcSizePtr > 0); + assert(source != NULL); + assert(opt != NULL); + assert(nbSearchesSeq >= 1); + + *srcSizePtr = 0; + if (inputSize < LZ4_minLength) goto _last_literals; + mflimit = iend - MFLIMIT; + matchlimit = iend - LASTLITERALS; + + /* Main Loop */ + while (ip <= mflimit) { + int const llen = LZ4HC_pricedLitlen((int)(ip - anchor)); + int cur, lastPos; + int lastSeqPos; /* position of the last sequence to encode */ + LZ4HC_match_t lastSeq; + + LZ4HC_match_t const firstMatch = LZ4HC_FindLongestMatch(ctx, ip, matchlimit, nbSearches, dict, favorDecSpeed); + if (firstMatch.len==0) { ip++; continue; } + + if (firstMatch.len > sufficientLen) { + /* good enough solution : immediate encoding */ + lastSeqPos = 0; + lastSeq = firstMatch; + goto encode; + } + + /* set prices for first positions (literals) */ + { int rPos; + for (rPos = 0 ; rPos < MINMATCH ; rPos++) + LZ4HC_setLiteral(opt, rPos, llen + rPos, LZ4HC_literalsPrice(llen + rPos)); + } + lastPos = LZ4HC_addMatch(opt, 0, firstMatch, 0, favorDecSpeed); + + /* check further positions */ + for (cur = 1; cur < lastPos; cur++) { + const BYTE* const curPtr = ip + cur; + LZ4HC_match_t newMatch; + int nbS = nbSearches; + + if (curPtr > mflimit) break; + DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u", + cur, opt[cur].price, opt[cur+1].price, cur+1); + /* not useful to search here if next position has same (or lower) cost */ + if (opt[cur+1].price <= opt[cur].price) continue; + /* favorDecSpeed : next position costs less than one more byte, the difference comes from sequences. + * A match starting here might save some : worth a cheaper search, if close to lastPos */ + if (favorDecSpeed && (opt[cur+1].price < opt[cur].price + LZ4HC_PRICE_SCALE)) { + if (lastPos - cur > LZ4HC_SEQ_SEARCH_DIST) continue; + nbS = nbSearchesSeq; + } + + newMatch = LZ4HC_FindLongestMatch(ctx, curPtr, matchlimit, nbS, dict, favorDecSpeed); + if (!newMatch.len) continue; + + if ( (newMatch.len > sufficientLen) + || (newMatch.len + cur >= LZ4_OPT_NUM) ) { + /* immediate encoding */ + lastSeqPos = cur; + lastSeq = newMatch; + goto encode; + } + + lastPos = LZ4HC_addMatch(opt, cur, newMatch, lastPos, favorDecSpeed); + } /* for (cur = 1; cur < lastPos; cur++) */ + + assert(lastPos < LZ4_OPT_NUM + TRAILING_LITERALS); + lastSeq.len = opt[lastPos].mlen; + lastSeq.off = opt[lastPos].off; + lastSeqPos = lastPos - lastSeq.len; + +encode: /* lastSeqPos and lastSeq must be set */ + assert(0 <= lastSeqPos && lastSeqPos < LZ4_OPT_NUM); + DEBUGLOG(6, "reverse traversal, looking for shortest path (lastSeqPos=%i)", lastSeqPos); + /* opt[pos] describes the step ending at pos. + * Walk the path backward, moving each step to its starting position instead */ + { int pos = lastSeqPos; + int mlen = lastSeq.len; + int off = lastSeq.off; + while (pos > 0) { + int const prevMlen = opt[pos].mlen; + int const prevOff = opt[pos].off; + opt[pos].mlen = mlen; + opt[pos].off = off; + mlen = prevMlen; + off = prevOff; + assert(0 < mlen && mlen <= pos); + pos -= mlen; + } + /* first step of the path; note : opt[0] isn't necessarily initialized */ + opt[0].mlen = mlen; + opt[0].off = off; + } + + /* encode all recorded sequences in order */ + { int rPos = 0; /* relative position (to ip) */ + while (rPos <= lastSeqPos) { + int const ml = opt[rPos].mlen; + int const offset = opt[rPos].off; + if (ml == 1) { ip++; rPos++; continue; } /* literal */ + rPos += ml; + assert(ml >= MINMATCH); + assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX)); + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, offset, limit, oend) ) { /* updates ip, op and anchor */ + if (limit != fillOutput) return 0; /* compression failed */ + LZ4HC_encodeLastSequence_fillOutput(UPDATABLE(ip, op, anchor), ml, offset, oend); + goto _last_literals; + } } } + } /* while (ip <= mflimit) */ + +_last_literals: + if (LZ4HC_encodeLastLiterals(UPDATABLE(ip, op, anchor), iend, limit, oend)) + return 0; /* not enough space in @dst */ + + /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); + return (int) ((char*)op-dst); +} static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, const char* const source, @@ -1829,308 +2181,58 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, int* srcSizePtr, int dstCapacity, int const nbSearches, - size_t sufficient_len, + int const nbSearchesSeq, + int const targetLength, const limitedOutput_directive limit, - int const fullUpdate, const dictCtx_directive dict, const HCfavor_e favorDecSpeed) { - int retval = 0; -#define TRAILING_LITERALS 3 + LZ4_STATIC_ASSERT(LZ4_STREAMHC_EXTSIZE == sizeof(LZ4_streamHC_t) + sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS)); + if (ctx->hasWorkspace) { /* extended state : the buffer follows it */ + LZ4HC_optimal_t* const opt = (LZ4HC_optimal_t*)(void*)((BYTE*)ctx + sizeof(LZ4_streamHC_t)); + return LZ4HC_compress_optimal_internal(ctx, source, dst, srcSizePtr, dstCapacity, + nbSearches, nbSearchesSeq, targetLength, limit, dict, favorDecSpeed, opt); + } + { /* plain state */ #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 LZ4HC_optimal_t* const opt = (LZ4HC_optimal_t*)ALLOC(sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS)); + int result; + if (opt == NULL) return 0; + result = LZ4HC_compress_optimal_internal(ctx, source, dst, srcSizePtr, dstCapacity, + nbSearches, nbSearchesSeq, targetLength, limit, dict, favorDecSpeed, opt); + FREEMEM(opt); + return result; #else - LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */ + LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which can be a bit large for some stacks... */ + return LZ4HC_compress_optimal_internal(ctx, source, dst, srcSizePtr, dstCapacity, + nbSearches, nbSearchesSeq, targetLength, limit, dict, favorDecSpeed, opt); #endif - - const BYTE* ip = (const BYTE*) source; - const BYTE* anchor = ip; - const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - BYTE* op = (BYTE*) dst; - BYTE* opSaved = (BYTE*) dst; - BYTE* oend = op + dstCapacity; - int ovml = MINMATCH; /* overflow - last sequence */ - int ovoff = 0; - - /* init */ -#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 - if (opt == NULL) goto _return_label; -#endif - DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity); - *srcSizePtr = 0; - if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ - if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; - - /* Main Loop */ - while (ip <= mflimit) { - int const llen = (int)(ip - anchor); - int best_mlen, best_off; - int cur, last_match_pos = 0; - - LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); - if (firstMatch.len==0) { ip++; continue; } - - if ((size_t)firstMatch.len > sufficient_len) { - /* good enough solution : immediate encoding */ - int const firstML = firstMatch.len; - opSaved = op; - if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, firstMatch.off, limit, oend) ) { /* updates ip, op and anchor */ - ovml = firstML; - ovoff = firstMatch.off; - goto _dest_overflow; - } - continue; - } - - /* set prices for first positions (literals) */ - { int rPos; - for (rPos = 0 ; rPos < MINMATCH ; rPos++) { - int const cost = LZ4HC_literalsPrice(llen + rPos); - opt[rPos].mlen = 1; - opt[rPos].off = 0; - opt[rPos].litlen = llen + rPos; - opt[rPos].price = cost; - DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", - rPos, cost, opt[rPos].litlen); - } } - /* set prices using initial match */ - { int const matchML = firstMatch.len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ - int const offset = firstMatch.off; - int mlen; - assert(matchML < LZ4_OPT_NUM); - for (mlen = MINMATCH ; mlen <= matchML ; mlen++) { - int const cost = LZ4HC_sequencePrice(llen, mlen); - opt[mlen].mlen = mlen; - opt[mlen].off = offset; - opt[mlen].litlen = llen; - opt[mlen].price = cost; - DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup", - mlen, cost, mlen); - } } - last_match_pos = firstMatch.len; - { int addLit; - for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { - opt[last_match_pos+addLit].mlen = 1; /* literal */ - opt[last_match_pos+addLit].off = 0; - opt[last_match_pos+addLit].litlen = addLit; - opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); - DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", - last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); - } } - - /* check further positions */ - for (cur = 1; cur < last_match_pos; cur++) { - const BYTE* const curPtr = ip + cur; - LZ4HC_match_t newMatch; - - if (curPtr > mflimit) break; - DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u", - cur, opt[cur].price, opt[cur+1].price, cur+1); - if (fullUpdate) { - /* not useful to search here if next position has same (or lower) cost */ - if ( (opt[cur+1].price <= opt[cur].price) - /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */ - && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) ) - continue; - } else { - /* not useful to search here if next position has same (or lower) cost */ - if (opt[cur+1].price <= opt[cur].price) continue; - } - - DEBUGLOG(7, "search at rPos:%u", cur); - if (fullUpdate) - newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); - else - /* only test matches of minimum length; slightly faster, but misses a few bytes */ - newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict, favorDecSpeed); - if (!newMatch.len) continue; - - if ( ((size_t)newMatch.len > sufficient_len) - || (newMatch.len + cur >= LZ4_OPT_NUM) ) { - /* immediate encoding */ - best_mlen = newMatch.len; - best_off = newMatch.off; - last_match_pos = cur + 1; - goto encode; - } - - /* before match : set price with literals at beginning */ - { int const baseLitlen = opt[cur].litlen; - int litlen; - for (litlen = 1; litlen < MINMATCH; litlen++) { - int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen); - int const pos = cur + litlen; - if (price < opt[pos].price) { - opt[pos].mlen = 1; /* literal */ - opt[pos].off = 0; - opt[pos].litlen = baseLitlen+litlen; - opt[pos].price = price; - DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", - pos, price, opt[pos].litlen); - } } } - - /* set prices using match at position = cur */ - { int const matchML = newMatch.len; - int ml = MINMATCH; - - assert(cur + newMatch.len < LZ4_OPT_NUM); - for ( ; ml <= matchML ; ml++) { - int const pos = cur + ml; - int const offset = newMatch.off; - int price; - int ll; - DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)", - pos, last_match_pos); - if (opt[cur].mlen == 1) { - ll = opt[cur].litlen; - price = ((cur > ll) ? opt[cur - ll].price : 0) - + LZ4HC_sequencePrice(ll, ml); - } else { - ll = 0; - price = opt[cur].price + LZ4HC_sequencePrice(0, ml); - } - - assert((U32)favorDecSpeed <= 1); - if (pos > last_match_pos+TRAILING_LITERALS - || price <= opt[pos].price - (int)favorDecSpeed) { - DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)", - pos, price, ml); - assert(pos < LZ4_OPT_NUM); - if ( (ml == matchML) /* last pos of last match */ - && (last_match_pos < pos) ) - last_match_pos = pos; - opt[pos].mlen = ml; - opt[pos].off = offset; - opt[pos].litlen = ll; - opt[pos].price = price; - } } } - /* complete following positions with literals */ - { int addLit; - for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { - opt[last_match_pos+addLit].mlen = 1; /* literal */ - opt[last_match_pos+addLit].off = 0; - opt[last_match_pos+addLit].litlen = addLit; - opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); - DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); - } } - } /* for (cur = 1; cur <= last_match_pos; cur++) */ - - assert(last_match_pos < LZ4_OPT_NUM + TRAILING_LITERALS); - best_mlen = opt[last_match_pos].mlen; - best_off = opt[last_match_pos].off; - cur = last_match_pos - best_mlen; - -encode: /* cur, last_match_pos, best_mlen, best_off must be set */ - assert(cur < LZ4_OPT_NUM); - assert(last_match_pos >= 1); /* == 1 when only one candidate */ - DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos); - { int candidate_pos = cur; - int selected_matchLength = best_mlen; - int selected_offset = best_off; - while (1) { /* from end to beginning */ - int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */ - int const next_offset = opt[candidate_pos].off; - DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength); - opt[candidate_pos].mlen = selected_matchLength; - opt[candidate_pos].off = selected_offset; - selected_matchLength = next_matchLength; - selected_offset = next_offset; - if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */ - assert(next_matchLength > 0); /* can be 1, means literal */ - candidate_pos -= next_matchLength; - } } - - /* encode all recorded sequences in order */ - { int rPos = 0; /* relative position (to ip) */ - while (rPos < last_match_pos) { - int const ml = opt[rPos].mlen; - int const offset = opt[rPos].off; - if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */ - rPos += ml; - assert(ml >= MINMATCH); - assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX)); - opSaved = op; - if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, offset, limit, oend) ) { /* updates ip, op and anchor */ - ovml = ml; - ovoff = offset; - goto _dest_overflow; - } } } - } /* while (ip <= mflimit) */ - -_last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ - size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; - size_t const totalSize = 1 + llAdd + lastRunSize; - if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ - if (limit && (op + totalSize > oend)) { - if (limit == limitedOutput) { /* Check output limit */ - retval = 0; - goto _return_label; - } - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (size_t)(oend - op) - 1 /*token*/; - llAdd = (lastRunSize + 256 - RUN_MASK) / 256; - lastRunSize -= llAdd; - } - DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); - ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = (RUN_MASK << ML_BITS); - for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize << ML_BITS); - } - LZ4_memcpy(op, anchor, lastRunSize); - op += lastRunSize; - } - - /* End */ - *srcSizePtr = (int) (((const char*)ip) - source); - retval = (int) ((char*)op-dst); - goto _return_label; - -_dest_overflow: -if (limit == fillOutput) { - /* Assumption : ip, anchor, ovml and ovref must be set correctly */ - size_t const ll = (size_t)(ip - anchor); - size_t const ll_addbytes = (ll + 240) / 255; - size_t const ll_totalCost = 1 + ll_addbytes + ll; - BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ - DEBUGLOG(6, "Last sequence overflowing (only %i bytes remaining)", (int)(oend-1-opSaved)); - op = opSaved; /* restore correct out pointer */ - if (op + ll_totalCost <= maxLitPos) { - /* ll validated; now adjust match length */ - size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); - size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); - assert(maxMlSize < INT_MAX); assert(ovml >= 0); - if ((size_t)ovml > maxMlSize) ovml = (int)maxMlSize; - if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + ovml >= MFLIMIT) { - DEBUGLOG(6, "Space to end : %i + ml (%i)", (int)((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1), ovml); - DEBUGLOG(6, "Before : ip = %p, anchor = %p", ip, anchor); - LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ovml, ovoff, notLimited, oend); - DEBUGLOG(6, "After : ip = %p, anchor = %p", ip, anchor); - } } - goto _last_literals; -} -_return_label: -#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 - if (opt) FREEMEM(opt); -#endif - return retval; + } } /*************************************************** -* Deprecated Functions +* Obsolete functions, removed from lz4hc.h (v1.11.0) +* still exported, for programs linked to previous versions ***************************************************/ -/* These functions currently generate deprecation warnings */ +LZ4LIB_API int LZ4_compressHC (const char* source, char* dest, int inputSize); +LZ4LIB_API int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4LIB_API int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); +LZ4LIB_API int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4LIB_API int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +LZ4LIB_API int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4LIB_API int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4LIB_API int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); +LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4LIB_API void* LZ4_createHC (const char* inputBuffer); +LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data); +LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data); +LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4LIB_API int LZ4_sizeofStreamStateHC(void); +LZ4LIB_API int LZ4_resetStreamStateHC(void* state, char* inputBuffer); /* Wrappers for deprecated compression functions */ int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } @@ -2175,14 +2277,17 @@ int LZ4_freeHC (void* LZ4HC_Data) } #endif +/* LZ4_compress_HC_continue() at level @cLevel : notably, restarts the stream after LZ4_slideInputBufferHC() */ int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel) { - return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, notLimited); + LZ4_setCompressionLevel((LZ4_streamHC_t*)LZ4HC_Data, cLevel); + return LZ4_compress_HC_continue((LZ4_streamHC_t*)LZ4HC_Data, src, dst, srcSize, LZ4_compressBound(srcSize)); } int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel) { - return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, dstCapacity, cLevel, limitedOutput); + LZ4_setCompressionLevel((LZ4_streamHC_t*)LZ4HC_Data, cLevel); + return LZ4_compress_HC_continue((LZ4_streamHC_t*)LZ4HC_Data, src, dst, srcSize, dstCapacity); } char* LZ4_slideInputBufferHC(void* LZ4HC_Data) diff --git a/public/common/tracy_lz4hc.hpp b/public/common/tracy_lz4hc.hpp index 695aa1e3..7457b753 100644 --- a/public/common/tracy_lz4hc.hpp +++ b/public/common/tracy_lz4hc.hpp @@ -1,7 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 Header File - Copyright (C) 2011-2020, Yann Collet. + Copyright (c) Yann Collet. All rights reserved. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -38,7 +38,6 @@ /* note : lz4hc requires lz4.h/lz4.c for compilation */ #include "tracy_lz4.hpp" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ - /* --- Useful constants --- */ #define LZ4HC_CLEVEL_MIN 2 #define LZ4HC_CLEVEL_DEFAULT 9 @@ -57,6 +56,10 @@ namespace tracy * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") * `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work. + * Levels 1 and 2 are identical : the fastest HC mode (since v1.10.0). + * Levels 3 to 9 trade speed for ratio (LZ4HC_CLEVEL_DEFAULT is 9). + * Levels from LZ4HC_CLEVEL_OPT_MIN (10) use the optimal parser, the slowest. + * Values < 1 select LZ4HC_CLEVEL_DEFAULT. * Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX. * @return : the number of bytes written into 'dst' * or 0 if compression fails. @@ -173,6 +176,13 @@ LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr const char* src, char* dst, int* srcSizePtr, int targetDstSize); +/*! LZ4_saveDictHC(): + * save history content (up to 64 KB) into a user-provided buffer @param safeBuffer. + * @return value is the size of dictionary effectively saved (<= MIN(maxDictSize, 64 KB)). + * This function is always successful, and expects its arguments to be valid. + * To this end, let's remind that (NULL,0) is valid, + * but (NULL,!0) is not, and will result in a segfault (or assert() depending on compilation mode). + */ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); @@ -195,8 +205,8 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in * a stream that has just been reset). * * The dictionary will remain attached to the working stream only for the - * current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the - * dictionary context association from the working stream. The dictionary + * current stream session. Calls to LZ4_resetStreamHC_fast() or LZ4_initStreamHC() + * will remove the dictionary context association from the working stream. The dictionary * stream (and source buffer) must remain in-place / accessible / unchanged * through the lifetime of the stream session. */ @@ -243,8 +253,8 @@ struct LZ4HC_CCtx_internal short compressionLevel; LZ4_i8 favorDecSpeed; /* favor decompression speed if this flag set, otherwise, favor compression ratio */ - LZ4_i8 dirty; /* stream has to be fully reset if this flag is set */ const LZ4HC_CCtx_internal* dictCtx; + LZ4_i8 hasWorkspace; /* the state is followed by the temporary buffer of levels >= 10 (LZ4_STREAMHC_EXTSIZE) */ }; #define LZ4_STREAMHC_MINSIZE 262200 /* static size, for inter-version compatibility */ @@ -278,46 +288,16 @@ LZ4LIB_API LZ4_streamHC_t* LZ4_initStreamHC(void* buffer, size_t size); **************************************/ /* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */ -/* deprecated compression functions */ -LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC (const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +/* Removed in v1.11.0, deprecated since v1.7.x (liblz4 still exports them, for linked programs) : + * LZ4_compressHC*(), LZ4_createHC(), LZ4_freeHC(), LZ4_sizeofStreamStateHC(), LZ4_resetStreamStateHC(), + * LZ4_slideInputBufferHC() */ -/* Obsolete streaming functions; degraded functionality; do not use! - * - * In order to perform streaming compression, these functions depended on data - * that is no longer tracked in the state. They have been preserved as well as - * possible: using them will still produce a correct output. However, use of - * LZ4_slideInputBufferHC() will truncate the history of the stream, rather - * than preserve a window-sized chunk of history. +/* LZ4_resetStreamHC() : deprecated (v1.11.0) + * Replaced by LZ4_initStreamHC(), which initializes a memory segment containing arbitrary garbage data, + * followed by LZ4_resetStreamHC_fast() to select the compression level. + * LZ4_resetStreamHC_fast() alone is the recommended function to start a new stream of blocks. */ -#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) -LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer); -LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data); -#endif -LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API int LZ4_sizeofStreamStateHC(void); -LZ4_DEPRECATED("use LZ4_initStreamHC() instead") LZ4LIB_API int LZ4_resetStreamStateHC(void* state, char* inputBuffer); - - -/* LZ4_resetStreamHC() is now replaced by LZ4_initStreamHC(). - * The intention is to emphasize the difference with LZ4_resetStreamHC_fast(), - * which is now the recommended function to start a new stream of blocks, - * but cannot be used to initialize a memory segment containing arbitrary garbage data. - * - * It is recommended to switch to LZ4_initStreamHC(). - * LZ4_resetStreamHC() will generate deprecation warnings in a future version. - */ -LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); +LZ4_DEPRECATED("use LZ4_initStreamHC() and LZ4_resetStreamHC_fast() instead") LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); } @@ -353,36 +333,42 @@ LZ4LIB_STATIC_API void LZ4_setCompressionLevel( /*! LZ4_favorDecompressionSpeed() : v1.8.2+ (experimental) * Opt. Parser will favor decompression speed over compression ratio. * Only applicable to levels >= LZ4HC_CLEVEL_OPT_MIN. + * The setting is preserved by LZ4_resetStreamHC_fast() and LZ4_loadDictHC(), + * it is reset by LZ4_initStreamHC(). */ LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed( LZ4_streamHC_t* LZ4_streamHCPtr, int favor); /*! LZ4_resetStreamHC_fast() : v1.9.0+ * When an LZ4_streamHC_t is known to be in a internally coherent state, - * it can often be prepared for a new compression with almost no work, only - * sometimes falling back to the full, expensive reset that is always required - * when the stream is in an indeterminate state (i.e., the reset performed by - * LZ4_resetStreamHC()). + * it can be prepared for a new compression with almost no work, instead of + * the full, expensive reset that is required when the stream is in an + * indeterminate state (i.e., the initialization performed by LZ4_initStreamHC()). * * LZ4_streamHCs are guaranteed to be in a valid state when: * - returned from LZ4_createStreamHC() - * - reset by LZ4_resetStreamHC() + * - initialized by LZ4_initStreamHC() * - memset(stream, 0, sizeof(LZ4_streamHC_t)) * - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast() - * - the stream was in a valid state and was then used in any compression call - * that returned success + * - the stream was in a valid state and was then used in any compression call, + * successful or not * - the stream was in an indeterminate state and was used in a compression * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that * returned success - * - * Note: - * A stream that was last used in a compression call that returned an error - * may be passed to this function. However, it will be fully reset, which will - * clear any existing history and settings from the context. */ LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast( LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); +/*! LZ4_STREAMHC_EXTSIZE : v1.11.0+ (experimental) + * Levels >= LZ4HC_CLEVEL_OPT_MIN employ a temporary buffer (~64 KB), allocated at each compression + * (or on stack, when lz4hc.c is compiled with LZ4HC_HEAPMODE=0). + * A state initialized by LZ4_initStreamHC() on a buffer of LZ4_STREAMHC_EXTSIZE bytes includes this buffer : + * its compressions allocate nothing. States created by LZ4_createStreamHC() include it. + * It is preserved by LZ4_resetStreamHC_fast() and LZ4_loadDictHC(), but not by LZ4_compress_HC_extStateHC(), + * which initializes @state as a state of LZ4_sizeofStateHC() bytes : use LZ4_compress_HC_extStateHC_fastReset() instead. + */ +#define LZ4_STREAMHC_EXTSIZE (LZ4_STREAMHC_MINSIZE + 65584) + /*! LZ4_compress_HC_extStateHC_fastReset() : * A variant of LZ4_compress_HC_extStateHC(). * @@ -392,7 +378,7 @@ LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast( * "correctly initialized"). From a high level, the difference is that this * function initializes the provided state with a call to * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a - * call to LZ4_resetStreamHC(). + * call to LZ4_initStreamHC(). */ LZ4LIB_STATIC_API int LZ4_compress_HC_extStateHC_fastReset ( void* state,