Commit c51288f1f6
Changed files (36)
stage1
zstd
lib
common
decompress
stage1/zstd/lib/common/bitstream.h
@@ -1,478 +0,0 @@
-/* ******************************************************************
- * bitstream
- * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
- *
- * You can contact the author at :
- * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
-****************************************************************** */
-#ifndef BITSTREAM_H_MODULE
-#define BITSTREAM_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-/*
-* This API consists of small unitary functions, which must be inlined for best performance.
-* Since link-time-optimization is not available for all compilers,
-* these functions are defined into a .h to be included.
-*/
-
-/*-****************************************
-* Dependencies
-******************************************/
-#include "mem.h" /* unaligned access routines */
-#include "compiler.h" /* UNLIKELY() */
-#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */
-#include "error_private.h" /* error codes and messages */
-
-
-/*=========================================
-* Target specific
-=========================================*/
-#ifndef ZSTD_NO_INTRINSICS
-# if defined(__BMI__) && defined(__GNUC__)
-# include <immintrin.h> /* support for bextr (experimental) */
-# elif defined(__ICCARM__)
-# include <intrinsics.h>
-# endif
-#endif
-
-#define STREAM_ACCUMULATOR_MIN_32 25
-#define STREAM_ACCUMULATOR_MIN_64 57
-#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
-
-
-/*-******************************************
-* bitStream encoding API (write forward)
-********************************************/
-/* bitStream can mix input from multiple sources.
- * A critical property of these streams is that they encode and decode in **reverse** direction.
- * So the first bit sequence you add will be the last to be read, like a LIFO stack.
- */
-typedef struct {
- size_t bitContainer;
- unsigned bitPos;
- char* startPtr;
- char* ptr;
- char* endPtr;
-} BIT_CStream_t;
-
-MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
-MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
-MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
-MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
-
-/* Start with initCStream, providing the size of buffer to write into.
-* bitStream will never write outside of this buffer.
-* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
-*
-* bits are first added to a local register.
-* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
-* Writing data into memory is an explicit operation, performed by the flushBits function.
-* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
-* After a flushBits, a maximum of 7 bits might still be stored into local register.
-*
-* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
-*
-* Last operation is to close the bitStream.
-* The function returns the final size of CStream in bytes.
-* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
-*/
-
-
-/*-********************************************
-* bitStream decoding API (read backward)
-**********************************************/
-typedef struct {
- size_t bitContainer;
- unsigned bitsConsumed;
- const char* ptr;
- const char* start;
- const char* limitPtr;
-} BIT_DStream_t;
-
-typedef enum { BIT_DStream_unfinished = 0,
- BIT_DStream_endOfBuffer = 1,
- BIT_DStream_completed = 2,
- BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */
- /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
-
-MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
-MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
-MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
-MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
-
-
-/* Start by invoking BIT_initDStream().
-* A chunk of the bitStream is then stored into a local register.
-* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
-* You can then retrieve bitFields stored into the local register, **in reverse order**.
-* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
-* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
-* Otherwise, it can be less than that, so proceed accordingly.
-* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
-*/
-
-
-/*-****************************************
-* unsafe API
-******************************************/
-MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
-/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
-
-MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
-/* unsafe version; does not check buffer overflow */
-
-MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
-/* faster, but works only if nbBits >= 1 */
-
-
-
-/*-**************************************************************
-* Internal functions
-****************************************************************/
-MEM_STATIC unsigned BIT_highbit32 (U32 val)
-{
- assert(val != 0);
- {
-# if defined(_MSC_VER) /* Visual */
-# if STATIC_BMI2 == 1
- return _lzcnt_u32(val) ^ 31;
-# else
- if (val != 0) {
- unsigned long r;
- _BitScanReverse(&r, val);
- return (unsigned)r;
- } else {
- /* Should not reach this code path */
- __assume(0);
- }
-# endif
-# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
- return __builtin_clz (val) ^ 31;
-# elif defined(__ICCARM__) /* IAR Intrinsic */
- return 31 - __CLZ(val);
-# else /* Software version */
- static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29,
- 11, 14, 16, 18, 22, 25, 3, 30,
- 8, 12, 20, 28, 15, 17, 24, 7,
- 19, 27, 23, 6, 26, 5, 4, 31 };
- U32 v = val;
- v |= v >> 1;
- v |= v >> 2;
- v |= v >> 4;
- v |= v >> 8;
- v |= v >> 16;
- return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
-# endif
- }
-}
-
-/*===== Local Constants =====*/
-static const unsigned BIT_mask[] = {
- 0, 1, 3, 7, 0xF, 0x1F,
- 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF,
- 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,
- 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
- 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
- 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */
-#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))
-
-/*-**************************************************************
-* bitStream encoding
-****************************************************************/
-/*! BIT_initCStream() :
- * `dstCapacity` must be > sizeof(size_t)
- * @return : 0 if success,
- * otherwise an error code (can be tested using ERR_isError()) */
-MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
- void* startPtr, size_t dstCapacity)
-{
- bitC->bitContainer = 0;
- bitC->bitPos = 0;
- bitC->startPtr = (char*)startPtr;
- bitC->ptr = bitC->startPtr;
- bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer);
- if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall);
- return 0;
-}
-
-/*! BIT_addBits() :
- * can add up to 31 bits into `bitC`.
- * Note : does not check for register overflow ! */
-MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
- size_t value, unsigned nbBits)
-{
- DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
- assert(nbBits < BIT_MASK_SIZE);
- assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
- bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
- bitC->bitPos += nbBits;
-}
-
-/*! BIT_addBitsFast() :
- * works only if `value` is _clean_,
- * meaning all high bits above nbBits are 0 */
-MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
- size_t value, unsigned nbBits)
-{
- assert((value>>nbBits) == 0);
- assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
- bitC->bitContainer |= value << bitC->bitPos;
- bitC->bitPos += nbBits;
-}
-
-/*! BIT_flushBitsFast() :
- * assumption : bitContainer has not overflowed
- * unsafe version; does not check buffer overflow */
-MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
-{
- size_t const nbBytes = bitC->bitPos >> 3;
- assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
- assert(bitC->ptr <= bitC->endPtr);
- MEM_writeLEST(bitC->ptr, bitC->bitContainer);
- bitC->ptr += nbBytes;
- bitC->bitPos &= 7;
- bitC->bitContainer >>= nbBytes*8;
-}
-
-/*! BIT_flushBits() :
- * assumption : bitContainer has not overflowed
- * safe version; check for buffer overflow, and prevents it.
- * note : does not signal buffer overflow.
- * overflow will be revealed later on using BIT_closeCStream() */
-MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
-{
- size_t const nbBytes = bitC->bitPos >> 3;
- assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
- assert(bitC->ptr <= bitC->endPtr);
- MEM_writeLEST(bitC->ptr, bitC->bitContainer);
- bitC->ptr += nbBytes;
- if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
- bitC->bitPos &= 7;
- bitC->bitContainer >>= nbBytes*8;
-}
-
-/*! BIT_closeCStream() :
- * @return : size of CStream, in bytes,
- * or 0 if it could not fit into dstBuffer */
-MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
-{
- BIT_addBitsFast(bitC, 1, 1); /* endMark */
- BIT_flushBits(bitC);
- if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
- return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
-}
-
-
-/*-********************************************************
-* bitStream decoding
-**********************************************************/
-/*! BIT_initDStream() :
- * Initialize a BIT_DStream_t.
- * `bitD` : a pointer to an already allocated BIT_DStream_t structure.
- * `srcSize` must be the *exact* size of the bitStream, in bytes.
- * @return : size of stream (== srcSize), or an errorCode if a problem is detected
- */
-MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
-{
- if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
-
- bitD->start = (const char*)srcBuffer;
- bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer);
-
- if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
- bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
- bitD->bitContainer = MEM_readLEST(bitD->ptr);
- { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
- bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
- if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
- } else {
- bitD->ptr = bitD->start;
- bitD->bitContainer = *(const BYTE*)(bitD->start);
- switch(srcSize)
- {
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
- ZSTD_FALLTHROUGH;
-
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
- ZSTD_FALLTHROUGH;
-
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
- ZSTD_FALLTHROUGH;
-
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
- ZSTD_FALLTHROUGH;
-
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
- ZSTD_FALLTHROUGH;
-
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
- ZSTD_FALLTHROUGH;
-
- default: break;
- }
- { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
- bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
- if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */
- }
- bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
- }
-
- return srcSize;
-}
-
-MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
-{
- return bitContainer >> start;
-}
-
-MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
-{
- U32 const regMask = sizeof(bitContainer)*8 - 1;
- /* if start > regMask, bitstream is corrupted, and result is undefined */
- assert(nbBits < BIT_MASK_SIZE);
- /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better
- * than accessing memory. When bmi2 instruction is not present, we consider
- * such cpus old (pre-Haswell, 2013) and their performance is not of that
- * importance.
- */
-#if defined(__x86_64__) || defined(_M_X86)
- return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
-#else
- return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
-#endif
-}
-
-MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
-{
-#if defined(STATIC_BMI2) && STATIC_BMI2 == 1
- return _bzhi_u64(bitContainer, nbBits);
-#else
- assert(nbBits < BIT_MASK_SIZE);
- return bitContainer & BIT_mask[nbBits];
-#endif
-}
-
-/*! BIT_lookBits() :
- * Provides next n bits from local register.
- * local register is not modified.
- * On 32-bits, maxNbBits==24.
- * On 64-bits, maxNbBits==56.
- * @return : value extracted */
-MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
-{
- /* arbitrate between double-shift and shift+mask */
-#if 1
- /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8,
- * bitstream is likely corrupted, and result is undefined */
- return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
-#else
- /* this code path is slower on my os-x laptop */
- U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
- return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
-#endif
-}
-
-/*! BIT_lookBitsFast() :
- * unsafe version; only works if nbBits >= 1 */
-MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
-{
- U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
- assert(nbBits >= 1);
- return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
-}
-
-MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
-{
- bitD->bitsConsumed += nbBits;
-}
-
-/*! BIT_readBits() :
- * Read (consume) next n bits from local register and update.
- * Pay attention to not read more than nbBits contained into local register.
- * @return : extracted value. */
-MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
-{
- size_t const value = BIT_lookBits(bitD, nbBits);
- BIT_skipBits(bitD, nbBits);
- return value;
-}
-
-/*! BIT_readBitsFast() :
- * unsafe version; only works only if nbBits >= 1 */
-MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
-{
- size_t const value = BIT_lookBitsFast(bitD, nbBits);
- assert(nbBits >= 1);
- BIT_skipBits(bitD, nbBits);
- return value;
-}
-
-/*! BIT_reloadDStreamFast() :
- * Similar to BIT_reloadDStream(), but with two differences:
- * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
- * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this
- * point you must use BIT_reloadDStream() to reload.
- */
-MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
-{
- if (UNLIKELY(bitD->ptr < bitD->limitPtr))
- return BIT_DStream_overflow;
- assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
- bitD->ptr -= bitD->bitsConsumed >> 3;
- bitD->bitsConsumed &= 7;
- bitD->bitContainer = MEM_readLEST(bitD->ptr);
- return BIT_DStream_unfinished;
-}
-
-/*! BIT_reloadDStream() :
- * Refill `bitD` from buffer previously set in BIT_initDStream() .
- * This function is safe, it guarantees it will not read beyond src buffer.
- * @return : status of `BIT_DStream_t` internal register.
- * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
-MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
-{
- if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */
- return BIT_DStream_overflow;
-
- if (bitD->ptr >= bitD->limitPtr) {
- return BIT_reloadDStreamFast(bitD);
- }
- if (bitD->ptr == bitD->start) {
- if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
- return BIT_DStream_completed;
- }
- /* start < ptr < limitPtr */
- { U32 nbBytes = bitD->bitsConsumed >> 3;
- BIT_DStream_status result = BIT_DStream_unfinished;
- if (bitD->ptr - nbBytes < bitD->start) {
- nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
- result = BIT_DStream_endOfBuffer;
- }
- bitD->ptr -= nbBytes;
- bitD->bitsConsumed -= nbBytes*8;
- bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */
- return result;
- }
-}
-
-/*! BIT_endOfDStream() :
- * @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
- */
-MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
-{
- return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
-}
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* BITSTREAM_H_MODULE */
stage1/zstd/lib/common/compiler.h
@@ -1,335 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef ZSTD_COMPILER_H
-#define ZSTD_COMPILER_H
-
-#include "portability_macros.h"
-
-/*-*******************************************************
-* Compiler specifics
-*********************************************************/
-/* force inlining */
-
-#if !defined(ZSTD_NO_INLINE)
-#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
-# define INLINE_KEYWORD inline
-#else
-# define INLINE_KEYWORD
-#endif
-
-#if defined(__GNUC__) || defined(__ICCARM__)
-# define FORCE_INLINE_ATTR __attribute__((always_inline))
-#elif defined(_MSC_VER)
-# define FORCE_INLINE_ATTR __forceinline
-#else
-# define FORCE_INLINE_ATTR
-#endif
-
-#else
-
-#define INLINE_KEYWORD
-#define FORCE_INLINE_ATTR
-
-#endif
-
-/**
- On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC).
- This explicitly marks such functions as __cdecl so that the code will still compile
- if a CC other than __cdecl has been made the default.
-*/
-#if defined(_MSC_VER)
-# define WIN_CDECL __cdecl
-#else
-# define WIN_CDECL
-#endif
-
-/**
- * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
- * parameters. They must be inlined for the compiler to eliminate the constant
- * branches.
- */
-#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
-/**
- * HINT_INLINE is used to help the compiler generate better code. It is *not*
- * used for "templates", so it can be tweaked based on the compilers
- * performance.
- *
- * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the
- * always_inline attribute.
- *
- * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline
- * attribute.
- */
-#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5
-# define HINT_INLINE static INLINE_KEYWORD
-#else
-# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR
-#endif
-
-/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
-#if defined(__GNUC__)
-# define UNUSED_ATTR __attribute__((unused))
-#else
-# define UNUSED_ATTR
-#endif
-
-/* force no inlining */
-#ifdef _MSC_VER
-# define FORCE_NOINLINE static __declspec(noinline)
-#else
-# if defined(__GNUC__) || defined(__ICCARM__)
-# define FORCE_NOINLINE static __attribute__((__noinline__))
-# else
-# define FORCE_NOINLINE static
-# endif
-#endif
-
-
-/* target attribute */
-#if defined(__GNUC__) || defined(__ICCARM__)
-# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
-#else
-# define TARGET_ATTRIBUTE(target)
-#endif
-
-/* Target attribute for BMI2 dynamic dispatch.
- * Enable lzcnt, bmi, and bmi2.
- * We test for bmi1 & bmi2. lzcnt is included in bmi1.
- */
-#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2")
-
-/* prefetch
- * can be disabled, by declaring NO_PREFETCH build macro */
-#if defined(NO_PREFETCH)
-# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
-# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
-#else
-# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
-# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
-# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
-# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
-# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
-# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
-# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
-# elif defined(__aarch64__)
-# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr)))
-# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr)))
-# else
-# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
-# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
-# endif
-#endif /* NO_PREFETCH */
-
-#define CACHELINE_SIZE 64
-
-#define PREFETCH_AREA(p, s) { \
- const char* const _ptr = (const char*)(p); \
- size_t const _size = (size_t)(s); \
- size_t _pos; \
- for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \
- PREFETCH_L2(_ptr + _pos); \
- } \
-}
-
-/* vectorization
- * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax,
- * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */
-#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__)
-# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5)
-# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
-# else
-# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")")
-# endif
-#else
-# define DONT_VECTORIZE
-#endif
-
-/* Tell the compiler that a branch is likely or unlikely.
- * Only use these macros if it causes the compiler to generate better code.
- * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc
- * and clang, please do.
- */
-#if defined(__GNUC__)
-#define LIKELY(x) (__builtin_expect((x), 1))
-#define UNLIKELY(x) (__builtin_expect((x), 0))
-#else
-#define LIKELY(x) (x)
-#define UNLIKELY(x) (x)
-#endif
-
-/* disable warnings */
-#ifdef _MSC_VER /* Visual Studio */
-# include <intrin.h> /* For Visual 2005 */
-# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
-# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
-# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
-# pragma warning(disable : 4324) /* disable: C4324: padded structure */
-#endif
-
-/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/
-#ifndef STATIC_BMI2
-# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))
-# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2
-# define STATIC_BMI2 1
-# endif
-# endif
-#endif
-
-#ifndef STATIC_BMI2
- #define STATIC_BMI2 0
-#endif
-
-/* compile time determination of SIMD support */
-#if !defined(ZSTD_NO_INTRINSICS)
-# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
-# define ZSTD_ARCH_X86_SSE2
-# endif
-# if defined(__ARM_NEON) || defined(_M_ARM64)
-# define ZSTD_ARCH_ARM_NEON
-# endif
-#
-# if defined(ZSTD_ARCH_X86_SSE2)
-# include <emmintrin.h>
-# elif defined(ZSTD_ARCH_ARM_NEON)
-# include <arm_neon.h>
-# endif
-#endif
-
-/* C-language Attributes are added in C23. */
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute)
-# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
-#else
-# define ZSTD_HAS_C_ATTRIBUTE(x) 0
-#endif
-
-/* Only use C++ attributes in C++. Some compilers report support for C++
- * attributes when compiling with C.
- */
-#if defined(__cplusplus) && defined(__has_cpp_attribute)
-# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
-#else
-# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0
-#endif
-
-/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute.
- * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough
- * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough
- * - Else: __attribute__((__fallthrough__))
- */
-#ifndef ZSTD_FALLTHROUGH
-# if ZSTD_HAS_C_ATTRIBUTE(fallthrough)
-# define ZSTD_FALLTHROUGH [[fallthrough]]
-# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough)
-# define ZSTD_FALLTHROUGH [[fallthrough]]
-# elif __has_attribute(__fallthrough__)
-/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon
- * gcc complains about: a label can only be part of a statement and a declaration is not a statement.
- */
-# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__))
-# else
-# define ZSTD_FALLTHROUGH
-# endif
-#endif
-
-/*-**************************************************************
-* Alignment check
-*****************************************************************/
-
-/* this test was initially positioned in mem.h,
- * but this file is removed (or replaced) for linux kernel
- * so it's now hosted in compiler.h,
- * which remains valid for both user & kernel spaces.
- */
-
-#ifndef ZSTD_ALIGNOF
-# if defined(__GNUC__) || defined(_MSC_VER)
-/* covers gcc, clang & MSVC */
-/* note : this section must come first, before C11,
- * due to a limitation in the kernel source generator */
-# define ZSTD_ALIGNOF(T) __alignof(T)
-
-# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
-/* C11 support */
-# include <stdalign.h>
-# define ZSTD_ALIGNOF(T) alignof(T)
-
-# else
-/* No known support for alignof() - imperfect backup */
-# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T))
-
-# endif
-#endif /* ZSTD_ALIGNOF */
-
-/*-**************************************************************
-* Sanitizer
-*****************************************************************/
-
-#if ZSTD_MEMORY_SANITIZER
-/* Not all platforms that support msan provide sanitizers/msan_interface.h.
- * We therefore declare the functions we need ourselves, rather than trying to
- * include the header file... */
-#include <stddef.h> /* size_t */
-#define ZSTD_DEPS_NEED_STDINT
-#include "zstd_deps.h" /* intptr_t */
-
-/* Make memory region fully initialized (without changing its contents). */
-void __msan_unpoison(const volatile void *a, size_t size);
-
-/* Make memory region fully uninitialized (without changing its contents).
- This is a legacy interface that does not update origin information. Use
- __msan_allocated_memory() instead. */
-void __msan_poison(const volatile void *a, size_t size);
-
-/* Returns the offset of the first (at least partially) poisoned byte in the
- memory range, or -1 if the whole range is good. */
-intptr_t __msan_test_shadow(const volatile void *x, size_t size);
-#endif
-
-#if ZSTD_ADDRESS_SANITIZER
-/* Not all platforms that support asan provide sanitizers/asan_interface.h.
- * We therefore declare the functions we need ourselves, rather than trying to
- * include the header file... */
-#include <stddef.h> /* size_t */
-
-/**
- * Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable.
- *
- * This memory must be previously allocated by your program. Instrumented
- * code is forbidden from accessing addresses in this region until it is
- * unpoisoned. This function is not guaranteed to poison the entire region -
- * it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan
- * alignment restrictions.
- *
- * \note This function is not thread-safe because no two threads can poison or
- * unpoison memory in the same memory region simultaneously.
- *
- * \param addr Start of memory region.
- * \param size Size of memory region. */
-void __asan_poison_memory_region(void const volatile *addr, size_t size);
-
-/**
- * Marks a memory region (<c>[addr, addr+size)</c>) as addressable.
- *
- * This memory must be previously allocated by your program. Accessing
- * addresses in this region is allowed until this region is poisoned again.
- * This function could unpoison a super-region of <c>[addr, addr+size)</c> due
- * to ASan alignment restrictions.
- *
- * \note This function is not thread-safe because no two threads can
- * poison or unpoison memory in the same memory region simultaneously.
- *
- * \param addr Start of memory region.
- * \param size Size of memory region. */
-void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
-#endif
-
-#endif /* ZSTD_COMPILER_H */
stage1/zstd/lib/common/cpu.h
@@ -1,213 +0,0 @@
-/*
- * Copyright (c) Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef ZSTD_COMMON_CPU_H
-#define ZSTD_COMMON_CPU_H
-
-/**
- * Implementation taken from folly/CpuId.h
- * https://github.com/facebook/folly/blob/master/folly/CpuId.h
- */
-
-#include "mem.h"
-
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-
-typedef struct {
- U32 f1c;
- U32 f1d;
- U32 f7b;
- U32 f7c;
-} ZSTD_cpuid_t;
-
-MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
- U32 f1c = 0;
- U32 f1d = 0;
- U32 f7b = 0;
- U32 f7c = 0;
-#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
- int reg[4];
- __cpuid((int*)reg, 0);
- {
- int const n = reg[0];
- if (n >= 1) {
- __cpuid((int*)reg, 1);
- f1c = (U32)reg[2];
- f1d = (U32)reg[3];
- }
- if (n >= 7) {
- __cpuidex((int*)reg, 7, 0);
- f7b = (U32)reg[1];
- f7c = (U32)reg[2];
- }
- }
-#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__)
- /* The following block like the normal cpuid branch below, but gcc
- * reserves ebx for use of its pic register so we must specially
- * handle the save and restore to avoid clobbering the register
- */
- U32 n;
- __asm__(
- "pushl %%ebx\n\t"
- "cpuid\n\t"
- "popl %%ebx\n\t"
- : "=a"(n)
- : "a"(0)
- : "ecx", "edx");
- if (n >= 1) {
- U32 f1a;
- __asm__(
- "pushl %%ebx\n\t"
- "cpuid\n\t"
- "popl %%ebx\n\t"
- : "=a"(f1a), "=c"(f1c), "=d"(f1d)
- : "a"(1));
- }
- if (n >= 7) {
- __asm__(
- "pushl %%ebx\n\t"
- "cpuid\n\t"
- "movl %%ebx, %%eax\n\t"
- "popl %%ebx"
- : "=a"(f7b), "=c"(f7c)
- : "a"(7), "c"(0)
- : "edx");
- }
-#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
- U32 n;
- __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx");
- if (n >= 1) {
- U32 f1a;
- __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx");
- }
- if (n >= 7) {
- U32 f7a;
- __asm__("cpuid"
- : "=a"(f7a), "=b"(f7b), "=c"(f7c)
- : "a"(7), "c"(0)
- : "edx");
- }
-#endif
- {
- ZSTD_cpuid_t cpuid;
- cpuid.f1c = f1c;
- cpuid.f1d = f1d;
- cpuid.f7b = f7b;
- cpuid.f7c = f7c;
- return cpuid;
- }
-}
-
-#define X(name, r, bit) \
- MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \
- return ((cpuid.r) & (1U << bit)) != 0; \
- }
-
-/* cpuid(1): Processor Info and Feature Bits. */
-#define C(name, bit) X(name, f1c, bit)
- C(sse3, 0)
- C(pclmuldq, 1)
- C(dtes64, 2)
- C(monitor, 3)
- C(dscpl, 4)
- C(vmx, 5)
- C(smx, 6)
- C(eist, 7)
- C(tm2, 8)
- C(ssse3, 9)
- C(cnxtid, 10)
- C(fma, 12)
- C(cx16, 13)
- C(xtpr, 14)
- C(pdcm, 15)
- C(pcid, 17)
- C(dca, 18)
- C(sse41, 19)
- C(sse42, 20)
- C(x2apic, 21)
- C(movbe, 22)
- C(popcnt, 23)
- C(tscdeadline, 24)
- C(aes, 25)
- C(xsave, 26)
- C(osxsave, 27)
- C(avx, 28)
- C(f16c, 29)
- C(rdrand, 30)
-#undef C
-#define D(name, bit) X(name, f1d, bit)
- D(fpu, 0)
- D(vme, 1)
- D(de, 2)
- D(pse, 3)
- D(tsc, 4)
- D(msr, 5)
- D(pae, 6)
- D(mce, 7)
- D(cx8, 8)
- D(apic, 9)
- D(sep, 11)
- D(mtrr, 12)
- D(pge, 13)
- D(mca, 14)
- D(cmov, 15)
- D(pat, 16)
- D(pse36, 17)
- D(psn, 18)
- D(clfsh, 19)
- D(ds, 21)
- D(acpi, 22)
- D(mmx, 23)
- D(fxsr, 24)
- D(sse, 25)
- D(sse2, 26)
- D(ss, 27)
- D(htt, 28)
- D(tm, 29)
- D(pbe, 31)
-#undef D
-
-/* cpuid(7): Extended Features. */
-#define B(name, bit) X(name, f7b, bit)
- B(bmi1, 3)
- B(hle, 4)
- B(avx2, 5)
- B(smep, 7)
- B(bmi2, 8)
- B(erms, 9)
- B(invpcid, 10)
- B(rtm, 11)
- B(mpx, 14)
- B(avx512f, 16)
- B(avx512dq, 17)
- B(rdseed, 18)
- B(adx, 19)
- B(smap, 20)
- B(avx512ifma, 21)
- B(pcommit, 22)
- B(clflushopt, 23)
- B(clwb, 24)
- B(avx512pf, 26)
- B(avx512er, 27)
- B(avx512cd, 28)
- B(sha, 29)
- B(avx512bw, 30)
- B(avx512vl, 31)
-#undef B
-#define C(name, bit) X(name, f7c, bit)
- C(prefetchwt1, 0)
- C(avx512vbmi, 1)
-#undef C
-
-#undef X
-
-#endif /* ZSTD_COMMON_CPU_H */
stage1/zstd/lib/common/debug.c
@@ -1,24 +0,0 @@
-/* ******************************************************************
- * debug
- * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
- *
- * You can contact the author at :
- * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
-****************************************************************** */
-
-
-/*
- * This module only hosts one global variable
- * which can be used to dynamically influence the verbosity of traces,
- * such as DEBUGLOG and RAWLOG
- */
-
-#include "debug.h"
-
-int g_debuglevel = DEBUGLEVEL;
stage1/zstd/lib/common/debug.h
@@ -1,107 +0,0 @@
-/* ******************************************************************
- * debug
- * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
- *
- * You can contact the author at :
- * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
-****************************************************************** */
-
-
-/*
- * The purpose of this header is to enable debug functions.
- * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,
- * and DEBUG_STATIC_ASSERT() for compile-time.
- *
- * By default, DEBUGLEVEL==0, which means run-time debug is disabled.
- *
- * Level 1 enables assert() only.
- * Starting level 2, traces can be generated and pushed to stderr.
- * The higher the level, the more verbose the traces.
- *
- * It's possible to dynamically adjust level using variable g_debug_level,
- * which is only declared if DEBUGLEVEL>=2,
- * and is a global variable, not multi-thread protected (use with care)
- */
-
-#ifndef DEBUG_H_12987983217
-#define DEBUG_H_12987983217
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* static assert is triggered at compile time, leaving no runtime artefact.
- * static assert only works with compile-time constants.
- * Also, this variant can only be used inside a function. */
-#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
-
-
-/* DEBUGLEVEL is expected to be defined externally,
- * typically through compiler command line.
- * Value must be a number. */
-#ifndef DEBUGLEVEL
-# define DEBUGLEVEL 0
-#endif
-
-
-/* recommended values for DEBUGLEVEL :
- * 0 : release mode, no debug, all run-time checks disabled
- * 1 : enables assert() only, no display
- * 2 : reserved, for currently active debug path
- * 3 : events once per object lifetime (CCtx, CDict, etc.)
- * 4 : events once per frame
- * 5 : events once per block
- * 6 : events once per sequence (verbose)
- * 7+: events at every position (*very* verbose)
- *
- * It's generally inconvenient to output traces > 5.
- * In which case, it's possible to selectively trigger high verbosity levels
- * by modifying g_debug_level.
- */
-
-#if (DEBUGLEVEL>=1)
-# define ZSTD_DEPS_NEED_ASSERT
-# include "zstd_deps.h"
-#else
-# ifndef assert /* assert may be already defined, due to prior #include <assert.h> */
-# define assert(condition) ((void)0) /* disable assert (default) */
-# endif
-#endif
-
-#if (DEBUGLEVEL>=2)
-# define ZSTD_DEPS_NEED_IO
-# include "zstd_deps.h"
-extern int g_debuglevel; /* the variable is only declared,
- it actually lives in debug.c,
- and is shared by the whole process.
- It's not thread-safe.
- It's useful when enabling very verbose levels
- on selective conditions (such as position in src) */
-
-# define RAWLOG(l, ...) { \
- if (l<=g_debuglevel) { \
- ZSTD_DEBUG_PRINT(__VA_ARGS__); \
- } }
-# define DEBUGLOG(l, ...) { \
- if (l<=g_debuglevel) { \
- ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \
- ZSTD_DEBUG_PRINT(" \n"); \
- } }
-#else
-# define RAWLOG(l, ...) {} /* disabled */
-# define DEBUGLOG(l, ...) {} /* disabled */
-#endif
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* DEBUG_H_12987983217 */
stage1/zstd/lib/common/entropy_common.c
@@ -1,368 +0,0 @@
-/* ******************************************************************
- * Common functions of New Generation Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
- *
- * You can contact the author at :
- * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
- * - Public forum : https://groups.google.com/forum/#!forum/lz4c
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
-****************************************************************** */
-
-/* *************************************
-* Dependencies
-***************************************/
-#include "mem.h"
-#include "error_private.h" /* ERR_*, ERROR */
-#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
-#include "fse.h"
-#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
-#include "huf.h"
-
-
-/*=== Version ===*/
-unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
-
-
-/*=== Error Management ===*/
-unsigned FSE_isError(size_t code) { return ERR_isError(code); }
-const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
-
-unsigned HUF_isError(size_t code) { return ERR_isError(code); }
-const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
-
-
-/*-**************************************************************
-* FSE NCount encoding-decoding
-****************************************************************/
-static U32 FSE_ctz(U32 val)
-{
- assert(val != 0);
- {
-# if defined(_MSC_VER) /* Visual */
- if (val != 0) {
- unsigned long r;
- _BitScanForward(&r, val);
- return (unsigned)r;
- } else {
- /* Should not reach this code path */
- __assume(0);
- }
-# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
- return __builtin_ctz(val);
-# elif defined(__ICCARM__) /* IAR Intrinsic */
- return __CTZ(val);
-# else /* Software version */
- U32 count = 0;
- while ((val & 1) == 0) {
- val >>= 1;
- ++count;
- }
- return count;
-# endif
- }
-}
-
-FORCE_INLINE_TEMPLATE
-size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
- const void* headerBuffer, size_t hbSize)
-{
- const BYTE* const istart = (const BYTE*) headerBuffer;
- const BYTE* const iend = istart + hbSize;
- const BYTE* ip = istart;
- int nbBits;
- int remaining;
- int threshold;
- U32 bitStream;
- int bitCount;
- unsigned charnum = 0;
- unsigned const maxSV1 = *maxSVPtr + 1;
- int previous0 = 0;
-
- if (hbSize < 8) {
- /* This function only works when hbSize >= 8 */
- char buffer[8] = {0};
- ZSTD_memcpy(buffer, headerBuffer, hbSize);
- { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
- buffer, sizeof(buffer));
- if (FSE_isError(countSize)) return countSize;
- if (countSize > hbSize) return ERROR(corruption_detected);
- return countSize;
- } }
- assert(hbSize >= 8);
-
- /* init */
- ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
- bitStream = MEM_readLE32(ip);
- nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
- if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
- bitStream >>= 4;
- bitCount = 4;
- *tableLogPtr = nbBits;
- remaining = (1<<nbBits)+1;
- threshold = 1<<nbBits;
- nbBits++;
-
- for (;;) {
- if (previous0) {
- /* Count the number of repeats. Each time the
- * 2-bit repeat code is 0b11 there is another
- * repeat.
- * Avoid UB by setting the high bit to 1.
- */
- int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1;
- while (repeats >= 12) {
- charnum += 3 * 12;
- if (LIKELY(ip <= iend-7)) {
- ip += 3;
- } else {
- bitCount -= (int)(8 * (iend - 7 - ip));
- bitCount &= 31;
- ip = iend - 4;
- }
- bitStream = MEM_readLE32(ip) >> bitCount;
- repeats = FSE_ctz(~bitStream | 0x80000000) >> 1;
- }
- charnum += 3 * repeats;
- bitStream >>= 2 * repeats;
- bitCount += 2 * repeats;
-
- /* Add the final repeat which isn't 0b11. */
- assert((bitStream & 3) < 3);
- charnum += bitStream & 3;
- bitCount += 2;
-
- /* This is an error, but break and return an error
- * at the end, because returning out of a loop makes
- * it harder for the compiler to optimize.
- */
- if (charnum >= maxSV1) break;
-
- /* We don't need to set the normalized count to 0
- * because we already memset the whole buffer to 0.
- */
-
- if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
- assert((bitCount >> 3) <= 3); /* For first condition to work */
- ip += bitCount>>3;
- bitCount &= 7;
- } else {
- bitCount -= (int)(8 * (iend - 4 - ip));
- bitCount &= 31;
- ip = iend - 4;
- }
- bitStream = MEM_readLE32(ip) >> bitCount;
- }
- {
- int const max = (2*threshold-1) - remaining;
- int count;
-
- if ((bitStream & (threshold-1)) < (U32)max) {
- count = bitStream & (threshold-1);
- bitCount += nbBits-1;
- } else {
- count = bitStream & (2*threshold-1);
- if (count >= threshold) count -= max;
- bitCount += nbBits;
- }
-
- count--; /* extra accuracy */
- /* When it matters (small blocks), this is a
- * predictable branch, because we don't use -1.
- */
- if (count >= 0) {
- remaining -= count;
- } else {
- assert(count == -1);
- remaining += count;
- }
- normalizedCounter[charnum++] = (short)count;
- previous0 = !count;
-
- assert(threshold > 1);
- if (remaining < threshold) {
- /* This branch can be folded into the
- * threshold update condition because we
- * know that threshold > 1.
- */
- if (remaining <= 1) break;
- nbBits = BIT_highbit32(remaining) + 1;
- threshold = 1 << (nbBits - 1);
- }
- if (charnum >= maxSV1) break;
-
- if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
- ip += bitCount>>3;
- bitCount &= 7;
- } else {
- bitCount -= (int)(8 * (iend - 4 - ip));
- bitCount &= 31;
- ip = iend - 4;
- }
- bitStream = MEM_readLE32(ip) >> bitCount;
- } }
- if (remaining != 1) return ERROR(corruption_detected);
- /* Only possible when there are too many zeros. */
- if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall);
- if (bitCount > 32) return ERROR(corruption_detected);
- *maxSVPtr = charnum-1;
-
- ip += (bitCount+7)>>3;
- return ip-istart;
-}
-
-/* Avoids the FORCE_INLINE of the _body() function. */
-static size_t FSE_readNCount_body_default(
- short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
- const void* headerBuffer, size_t hbSize)
-{
- return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
-}
-
-#if DYNAMIC_BMI2
-BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2(
- short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
- const void* headerBuffer, size_t hbSize)
-{
- return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
-}
-#endif
-
-size_t FSE_readNCount_bmi2(
- short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
- const void* headerBuffer, size_t hbSize, int bmi2)
-{
-#if DYNAMIC_BMI2
- if (bmi2) {
- return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
- }
-#endif
- (void)bmi2;
- return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
-}
-
-size_t FSE_readNCount(
- short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
- const void* headerBuffer, size_t hbSize)
-{
- return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0);
-}
-
-
-/*! HUF_readStats() :
- Read compact Huffman tree, saved by HUF_writeCTable().
- `huffWeight` is destination buffer.
- `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
- @return : size read from `src` , or an error Code .
- Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
-*/
-size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
- U32* nbSymbolsPtr, U32* tableLogPtr,
- const void* src, size_t srcSize)
-{
- U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
- return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0);
-}
-
-FORCE_INLINE_TEMPLATE size_t
-HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
- U32* nbSymbolsPtr, U32* tableLogPtr,
- const void* src, size_t srcSize,
- void* workSpace, size_t wkspSize,
- int bmi2)
-{
- U32 weightTotal;
- const BYTE* ip = (const BYTE*) src;
- size_t iSize;
- size_t oSize;
-
- if (!srcSize) return ERROR(srcSize_wrong);
- iSize = ip[0];
- /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
-
- if (iSize >= 128) { /* special header */
- oSize = iSize - 127;
- iSize = ((oSize+1)/2);
- if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
- if (oSize >= hwSize) return ERROR(corruption_detected);
- ip += 1;
- { U32 n;
- for (n=0; n<oSize; n+=2) {
- huffWeight[n] = ip[n/2] >> 4;
- huffWeight[n+1] = ip[n/2] & 15;
- } } }
- else { /* header compressed with FSE (normal case) */
- if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
- /* max (hwSize-1) values decoded, as last one is implied */
- oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2);
- if (FSE_isError(oSize)) return oSize;
- }
-
- /* collect weight stats */
- ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
- weightTotal = 0;
- { U32 n; for (n=0; n<oSize; n++) {
- if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
- rankStats[huffWeight[n]]++;
- weightTotal += (1 << huffWeight[n]) >> 1;
- } }
- if (weightTotal == 0) return ERROR(corruption_detected);
-
- /* get last non-null symbol weight (implied, total must be 2^n) */
- { U32 const tableLog = BIT_highbit32(weightTotal) + 1;
- if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
- *tableLogPtr = tableLog;
- /* determine last weight */
- { U32 const total = 1 << tableLog;
- U32 const rest = total - weightTotal;
- U32 const verif = 1 << BIT_highbit32(rest);
- U32 const lastWeight = BIT_highbit32(rest) + 1;
- if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
- huffWeight[oSize] = (BYTE)lastWeight;
- rankStats[lastWeight]++;
- } }
-
- /* check tree construction validity */
- if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
-
- /* results */
- *nbSymbolsPtr = (U32)(oSize+1);
- return iSize+1;
-}
-
-/* Avoids the FORCE_INLINE of the _body() function. */
-static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats,
- U32* nbSymbolsPtr, U32* tableLogPtr,
- const void* src, size_t srcSize,
- void* workSpace, size_t wkspSize)
-{
- return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0);
-}
-
-#if DYNAMIC_BMI2
-static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats,
- U32* nbSymbolsPtr, U32* tableLogPtr,
- const void* src, size_t srcSize,
- void* workSpace, size_t wkspSize)
-{
- return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1);
-}
-#endif
-
-size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
- U32* nbSymbolsPtr, U32* tableLogPtr,
- const void* src, size_t srcSize,
- void* workSpace, size_t wkspSize,
- int bmi2)
-{
-#if DYNAMIC_BMI2
- if (bmi2) {
- return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
- }
-#endif
- (void)bmi2;
- return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
-}
stage1/zstd/lib/common/error_private.c
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-/* The purpose of this file is to have a single list of error strings embedded in binary */
-
-#include "error_private.h"
-
-const char* ERR_getErrorString(ERR_enum code)
-{
-#ifdef ZSTD_STRIP_ERROR_STRINGS
- (void)code;
- return "Error strings stripped";
-#else
- static const char* const notErrorCode = "Unspecified error code";
- switch( code )
- {
- case PREFIX(no_error): return "No error detected";
- case PREFIX(GENERIC): return "Error (generic)";
- case PREFIX(prefix_unknown): return "Unknown frame descriptor";
- case PREFIX(version_unsupported): return "Version not supported";
- case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
- case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
- case PREFIX(corruption_detected): return "Corrupted block detected";
- case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
- case PREFIX(parameter_unsupported): return "Unsupported parameter";
- case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
- case PREFIX(init_missing): return "Context should be init first";
- case PREFIX(memory_allocation): return "Allocation error : not enough memory";
- case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";
- case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
- case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
- case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
- case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
- case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
- case PREFIX(dictionary_wrong): return "Dictionary mismatch";
- case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
- case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
- case PREFIX(srcSize_wrong): return "Src size is incorrect";
- case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
- /* following error codes are not stable and may be removed or changed in a future version */
- case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
- case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
- case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
- case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
- case PREFIX(maxCode):
- default: return notErrorCode;
- }
-#endif
-}
stage1/zstd/lib/common/error_private.h
@@ -1,159 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-/* Note : this module is expected to remain private, do not expose it */
-
-#ifndef ERROR_H_MODULE
-#define ERROR_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* ****************************************
-* Dependencies
-******************************************/
-#include "../zstd_errors.h" /* enum list */
-#include "compiler.h"
-#include "debug.h"
-#include "zstd_deps.h" /* size_t */
-
-
-/* ****************************************
-* Compiler-specific
-******************************************/
-#if defined(__GNUC__)
-# define ERR_STATIC static __attribute__((unused))
-#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define ERR_STATIC static inline
-#elif defined(_MSC_VER)
-# define ERR_STATIC static __inline
-#else
-# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
-#endif
-
-
-/*-****************************************
-* Customization (error_public.h)
-******************************************/
-typedef ZSTD_ErrorCode ERR_enum;
-#define PREFIX(name) ZSTD_error_##name
-
-
-/*-****************************************
-* Error codes handling
-******************************************/
-#undef ERROR /* already defined on Visual Studio */
-#define ERROR(name) ZSTD_ERROR(name)
-#define ZSTD_ERROR(name) ((size_t)-PREFIX(name))
-
-ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
-
-ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
-
-/* check and forward error code */
-#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
-#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
-
-
-/*-****************************************
-* Error Strings
-******************************************/
-
-const char* ERR_getErrorString(ERR_enum code); /* error_private.c */
-
-ERR_STATIC const char* ERR_getErrorName(size_t code)
-{
- return ERR_getErrorString(ERR_getErrorCode(code));
-}
-
-/**
- * Ignore: this is an internal helper.
- *
- * This is a helper function to help force C99-correctness during compilation.
- * Under strict compilation modes, variadic macro arguments can't be empty.
- * However, variadic function arguments can be. Using a function therefore lets
- * us statically check that at least one (string) argument was passed,
- * independent of the compilation flags.
- */
-static INLINE_KEYWORD UNUSED_ATTR
-void _force_has_format_string(const char *format, ...) {
- (void)format;
-}
-
-/**
- * Ignore: this is an internal helper.
- *
- * We want to force this function invocation to be syntactically correct, but
- * we don't want to force runtime evaluation of its arguments.
- */
-#define _FORCE_HAS_FORMAT_STRING(...) \
- if (0) { \
- _force_has_format_string(__VA_ARGS__); \
- }
-
-#define ERR_QUOTE(str) #str
-
-/**
- * Return the specified error if the condition evaluates to true.
- *
- * In debug modes, prints additional information.
- * In order to do that (particularly, printing the conditional that failed),
- * this can't just wrap RETURN_ERROR().
- */
-#define RETURN_ERROR_IF(cond, err, ...) \
- if (cond) { \
- RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \
- __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \
- _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
- RAWLOG(3, ": " __VA_ARGS__); \
- RAWLOG(3, "\n"); \
- return ERROR(err); \
- }
-
-/**
- * Unconditionally return the specified error.
- *
- * In debug modes, prints additional information.
- */
-#define RETURN_ERROR(err, ...) \
- do { \
- RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \
- __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \
- _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
- RAWLOG(3, ": " __VA_ARGS__); \
- RAWLOG(3, "\n"); \
- return ERROR(err); \
- } while(0);
-
-/**
- * If the provided expression evaluates to an error code, returns that error code.
- *
- * In debug modes, prints additional information.
- */
-#define FORWARD_IF_ERROR(err, ...) \
- do { \
- size_t const err_code = (err); \
- if (ERR_isError(err_code)) { \
- RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \
- __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \
- _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
- RAWLOG(3, ": " __VA_ARGS__); \
- RAWLOG(3, "\n"); \
- return err_code; \
- } \
- } while(0);
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_H_MODULE */
stage1/zstd/lib/common/fse.h
@@ -1,717 +0,0 @@
-/* ******************************************************************
- * FSE : Finite State Entropy codec
- * Public Prototypes declaration
- * Copyright (c) Yann Collet, Facebook, Inc.
- *
- * You can contact the author at :
- * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
-****************************************************************** */
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-#ifndef FSE_H
-#define FSE_H
-
-
-/*-*****************************************
-* Dependencies
-******************************************/
-#include "zstd_deps.h" /* size_t, ptrdiff_t */
-
-
-/*-*****************************************
-* FSE_PUBLIC_API : control library symbols visibility
-******************************************/
-#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
-# define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
-#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
-# define FSE_PUBLIC_API __declspec(dllexport)
-#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
-# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
-#else
-# define FSE_PUBLIC_API
-#endif
-
-/*------ Version ------*/
-#define FSE_VERSION_MAJOR 0
-#define FSE_VERSION_MINOR 9
-#define FSE_VERSION_RELEASE 0
-
-#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
-#define FSE_QUOTE(str) #str
-#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
-#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
-
-#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
-FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
-
-
-/*-****************************************
-* FSE simple functions
-******************************************/
-/*! FSE_compress() :
- Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
- 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
- @return : size of compressed data (<= dstCapacity).
- Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
- if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
- if FSE_isError(return), compression failed (more details using FSE_getErrorName())
-*/
-FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
- const void* src, size_t srcSize);
-
-/*! FSE_decompress():
- Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
- into already allocated destination buffer 'dst', of size 'dstCapacity'.
- @return : size of regenerated data (<= maxDstSize),
- or an error code, which can be tested using FSE_isError() .
-
- ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
- Why ? : making this distinction requires a header.
- Header management is intentionally delegated to the user layer, which can better manage special cases.
-*/
-FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity,
- const void* cSrc, size_t cSrcSize);
-
-
-/*-*****************************************
-* Tool functions
-******************************************/
-FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
-
-/* Error Management */
-FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
-FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
-
-
-/*-*****************************************
-* FSE advanced functions
-******************************************/
-/*! FSE_compress2() :
- Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
- Both parameters can be defined as '0' to mean : use default value
- @return : size of compressed data
- Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
- if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
- if FSE_isError(return), it's an error code.
-*/
-FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-
-
-/*-*****************************************
-* FSE detailed API
-******************************************/
-/*!
-FSE_compress() does the following:
-1. count symbol occurrence from source[] into table count[] (see hist.h)
-2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
-3. save normalized counters to memory buffer using writeNCount()
-4. build encoding table 'CTable' from normalized counters
-5. encode the data stream using encoding table 'CTable'
-
-FSE_decompress() does the following:
-1. read normalized counters with readNCount()
-2. build decoding table 'DTable' from normalized counters
-3. decode the data stream using decoding table 'DTable'
-
-The following API allows targeting specific sub-functions for advanced tasks.
-For example, it's possible to compress several blocks using the same 'CTable',
-or to save and provide normalized distribution using external method.
-*/
-
-/* *** COMPRESSION *** */
-
-/*! FSE_optimalTableLog():
- dynamically downsize 'tableLog' when conditions are met.
- It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
- @return : recommended tableLog (necessarily <= 'maxTableLog') */
-FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
-
-/*! FSE_normalizeCount():
- normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
- 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
- useLowProbCount is a boolean parameter which trades off compressed size for
- faster header decoding. When it is set to 1, the compressed data will be slightly
- smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be
- faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0
- is a good default, since header deserialization makes a big speed difference.
- Otherwise, useLowProbCount=1 is a good default, since the speed difference is small.
- @return : tableLog,
- or an errorCode, which can be tested using FSE_isError() */
-FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
- const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount);
-
-/*! FSE_NCountWriteBound():
- Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
- Typically useful for allocation purpose. */
-FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
-
-/*! FSE_writeNCount():
- Compactly save 'normalizedCounter' into 'buffer'.
- @return : size of the compressed table,
- or an errorCode, which can be tested using FSE_isError(). */
-FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
- const short* normalizedCounter,
- unsigned maxSymbolValue, unsigned tableLog);
-
-/*! Constructor and Destructor of FSE_CTable.
- Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
-typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
-FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
-FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct);
-
-/*! FSE_buildCTable():
- Builds `ct`, which must be already allocated, using FSE_createCTable().
- @return : 0, or an errorCode, which can be tested using FSE_isError() */
-FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
-
-/*! FSE_compress_usingCTable():
- Compress `src` using `ct` into `dst` which must be already allocated.
- @return : size of compressed data (<= `dstCapacity`),
- or 0 if compressed data could not fit into `dst`,
- or an errorCode, which can be tested using FSE_isError() */
-FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
-
-/*!
-Tutorial :
-----------
-The first step is to count all symbols. FSE_count() does this job very fast.
-Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
-'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
-maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
-FSE_count() will return the number of occurrence of the most frequent symbol.
-This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
-If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
-
-The next step is to normalize the frequencies.
-FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
-It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
-You can use 'tableLog'==0 to mean "use default tableLog value".
-If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
-which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
-
-The result of FSE_normalizeCount() will be saved into a table,
-called 'normalizedCounter', which is a table of signed short.
-'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
-The return value is tableLog if everything proceeded as expected.
-It is 0 if there is a single symbol within distribution.
-If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
-
-'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
-'buffer' must be already allocated.
-For guaranteed success, buffer size must be at least FSE_headerBound().
-The result of the function is the number of bytes written into 'buffer'.
-If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
-
-'normalizedCounter' can then be used to create the compression table 'CTable'.
-The space required by 'CTable' must be already allocated, using FSE_createCTable().
-You can then use FSE_buildCTable() to fill 'CTable'.
-If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
-
-'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
-Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
-The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
-If it returns '0', compressed data could not fit into 'dst'.
-If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
-*/
-
-
-/* *** DECOMPRESSION *** */
-
-/*! FSE_readNCount():
- Read compactly saved 'normalizedCounter' from 'rBuffer'.
- @return : size read from 'rBuffer',
- or an errorCode, which can be tested using FSE_isError().
- maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
-FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
- unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
- const void* rBuffer, size_t rBuffSize);
-
-/*! FSE_readNCount_bmi2():
- * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.
- */
-FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
- unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
- const void* rBuffer, size_t rBuffSize, int bmi2);
-
-/*! Constructor and Destructor of FSE_DTable.
- Note that its size depends on 'tableLog' */
-typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
-FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
-FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt);
-
-/*! FSE_buildDTable():
- Builds 'dt', which must be already allocated, using FSE_createDTable().
- return : 0, or an errorCode, which can be tested using FSE_isError() */
-FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
-
-/*! FSE_decompress_usingDTable():
- Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
- into `dst` which must be already allocated.
- @return : size of regenerated data (necessarily <= `dstCapacity`),
- or an errorCode, which can be tested using FSE_isError() */
-FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
-
-/*!
-Tutorial :
-----------
-(Note : these functions only decompress FSE-compressed blocks.
- If block is uncompressed, use memcpy() instead
- If block is a single repeated byte, use memset() instead )
-
-The first step is to obtain the normalized frequencies of symbols.
-This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
-'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
-In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
-or size the table to handle worst case situations (typically 256).
-FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
-The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
-Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
-If there is an error, the function will return an error code, which can be tested using FSE_isError().
-
-The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
-This is performed by the function FSE_buildDTable().
-The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
-If there is an error, the function will return an error code, which can be tested using FSE_isError().
-
-`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
-`cSrcSize` must be strictly correct, otherwise decompression will fail.
-FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
-If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
-*/
-
-#endif /* FSE_H */
-
-#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
-#define FSE_H_FSE_STATIC_LINKING_ONLY
-
-/* *** Dependency *** */
-#include "bitstream.h"
-
-
-/* *****************************************
-* Static allocation
-*******************************************/
-/* FSE buffer bounds */
-#define FSE_NCOUNTBOUND 512
-#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */)
-#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
-
-/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
-#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2))
-#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog)))
-
-/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */
-#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))
-#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable))
-
-
-/* *****************************************
- * FSE advanced API
- ***************************************** */
-
-unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
-/**< same as FSE_optimalTableLog(), which used `minus==2` */
-
-/* FSE_compress_wksp() :
- * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
- * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
- */
-#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
-size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
-
-size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
-/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
-
-size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
-/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
-
-/* FSE_buildCTable_wksp() :
- * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
- * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`.
- * See FSE_buildCTable_wksp() for breakdown of workspace usage.
- */
-#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */)
-#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog))
-size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
-
-#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8)
-#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned))
-FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
-/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
-
-size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
-/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
-
-size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
-/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
-
-#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
-#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
-size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize);
-/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */
-
-size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
-/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */
-
-typedef enum {
- FSE_repeat_none, /**< Cannot use the previous table */
- FSE_repeat_check, /**< Can use the previous table but it must be checked */
- FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */
- } FSE_repeat;
-
-/* *****************************************
-* FSE symbol compression API
-*******************************************/
-/*!
- This API consists of small unitary functions, which highly benefit from being inlined.
- Hence their body are included in next section.
-*/
-typedef struct {
- ptrdiff_t value;
- const void* stateTable;
- const void* symbolTT;
- unsigned stateLog;
-} FSE_CState_t;
-
-static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
-
-static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
-
-static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
-
-/**<
-These functions are inner components of FSE_compress_usingCTable().
-They allow the creation of custom streams, mixing multiple tables and bit sources.
-
-A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
-So the first symbol you will encode is the last you will decode, like a LIFO stack.
-
-You will need a few variables to track your CStream. They are :
-
-FSE_CTable ct; // Provided by FSE_buildCTable()
-BIT_CStream_t bitStream; // bitStream tracking structure
-FSE_CState_t state; // State tracking structure (can have several)
-
-
-The first thing to do is to init bitStream and state.
- size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
- FSE_initCState(&state, ct);
-
-Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
-You can then encode your input data, byte after byte.
-FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
-Remember decoding will be done in reverse direction.
- FSE_encodeByte(&bitStream, &state, symbol);
-
-At any time, you can also add any bit sequence.
-Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
- BIT_addBits(&bitStream, bitField, nbBits);
-
-The above methods don't commit data to memory, they just store it into local register, for speed.
-Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
-Writing data to memory is a manual operation, performed by the flushBits function.
- BIT_flushBits(&bitStream);
-
-Your last FSE encoding operation shall be to flush your last state value(s).
- FSE_flushState(&bitStream, &state);
-
-Finally, you must close the bitStream.
-The function returns the size of CStream in bytes.
-If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
-If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
- size_t size = BIT_closeCStream(&bitStream);
-*/
-
-
-/* *****************************************
-* FSE symbol decompression API
-*******************************************/
-typedef struct {
- size_t state;
- const void* table; /* precise table may vary, depending on U16 */
-} FSE_DState_t;
-
-
-static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
-
-static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
-
-static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
-
-/**<
-Let's now decompose FSE_decompress_usingDTable() into its unitary components.
-You will decode FSE-encoded symbols from the bitStream,
-and also any other bitFields you put in, **in reverse order**.
-
-You will need a few variables to track your bitStream. They are :
-
-BIT_DStream_t DStream; // Stream context
-FSE_DState_t DState; // State context. Multiple ones are possible
-FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
-
-The first thing to do is to init the bitStream.
- errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
-
-You should then retrieve your initial state(s)
-(in reverse flushing order if you have several ones) :
- errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
-
-You can then decode your data, symbol after symbol.
-For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
-Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
- unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
-
-You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
-Note : maximum allowed nbBits is 25, for 32-bits compatibility
- size_t bitField = BIT_readBits(&DStream, nbBits);
-
-All above operations only read from local register (which size depends on size_t).
-Refueling the register from memory is manually performed by the reload method.
- endSignal = FSE_reloadDStream(&DStream);
-
-BIT_reloadDStream() result tells if there is still some more data to read from DStream.
-BIT_DStream_unfinished : there is still some data left into the DStream.
-BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
-BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
-BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
-
-When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
-to properly detect the exact end of stream.
-After each decoded symbol, check if DStream is fully consumed using this simple test :
- BIT_reloadDStream(&DStream) >= BIT_DStream_completed
-
-When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
-Checking if DStream has reached its end is performed by :
- BIT_endOfDStream(&DStream);
-Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
- FSE_endOfDState(&DState);
-*/
-
-
-/* *****************************************
-* FSE unsafe API
-*******************************************/
-static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
-/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
-
-
-/* *****************************************
-* Implementation of inlined functions
-*******************************************/
-typedef struct {
- int deltaFindState;
- U32 deltaNbBits;
-} FSE_symbolCompressionTransform; /* total 8 bytes */
-
-MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
-{
- const void* ptr = ct;
- const U16* u16ptr = (const U16*) ptr;
- const U32 tableLog = MEM_read16(ptr);
- statePtr->value = (ptrdiff_t)1<<tableLog;
- statePtr->stateTable = u16ptr+2;
- statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
- statePtr->stateLog = tableLog;
-}
-
-
-/*! FSE_initCState2() :
-* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
-* uses the smallest state value possible, saving the cost of this symbol */
-MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
-{
- FSE_initCState(statePtr, ct);
- { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
- const U16* stateTable = (const U16*)(statePtr->stateTable);
- U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
- statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
- statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
- }
-}
-
-MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
-{
- FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
- const U16* const stateTable = (const U16*)(statePtr->stateTable);
- U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
- BIT_addBits(bitC, statePtr->value, nbBitsOut);
- statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
-}
-
-MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
-{
- BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
- BIT_flushBits(bitC);
-}
-
-
-/* FSE_getMaxNbBits() :
- * Approximate maximum cost of a symbol, in bits.
- * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
- * note 1 : assume symbolValue is valid (<= maxSymbolValue)
- * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
-MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
-{
- const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
- return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
-}
-
-/* FSE_bitCost() :
- * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
- * note 1 : assume symbolValue is valid (<= maxSymbolValue)
- * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
-MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
-{
- const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
- U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
- U32 const threshold = (minNbBits+1) << 16;
- assert(tableLog < 16);
- assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */
- { U32 const tableSize = 1 << tableLog;
- U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
- U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */
- U32 const bitMultiplier = 1 << accuracyLog;
- assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
- assert(normalizedDeltaFromThreshold <= bitMultiplier);
- return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
- }
-}
-
-
-/* ====== Decompression ====== */
-
-typedef struct {
- U16 tableLog;
- U16 fastMode;
-} FSE_DTableHeader; /* sizeof U32 */
-
-typedef struct
-{
- unsigned short newState;
- unsigned char symbol;
- unsigned char nbBits;
-} FSE_decode_t; /* size == U32 */
-
-MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
-{
- const void* ptr = dt;
- const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
- DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
- BIT_reloadDStream(bitD);
- DStatePtr->table = dt + 1;
-}
-
-MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
-{
- FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
- return DInfo.symbol;
-}
-
-MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
-{
- FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
- U32 const nbBits = DInfo.nbBits;
- size_t const lowBits = BIT_readBits(bitD, nbBits);
- DStatePtr->state = DInfo.newState + lowBits;
-}
-
-MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
-{
- FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
- U32 const nbBits = DInfo.nbBits;
- BYTE const symbol = DInfo.symbol;
- size_t const lowBits = BIT_readBits(bitD, nbBits);
-
- DStatePtr->state = DInfo.newState + lowBits;
- return symbol;
-}
-
-/*! FSE_decodeSymbolFast() :
- unsafe, only works if no symbol has a probability > 50% */
-MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
-{
- FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
- U32 const nbBits = DInfo.nbBits;
- BYTE const symbol = DInfo.symbol;
- size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
-
- DStatePtr->state = DInfo.newState + lowBits;
- return symbol;
-}
-
-MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
-{
- return DStatePtr->state == 0;
-}
-
-
-
-#ifndef FSE_COMMONDEFS_ONLY
-
-/* **************************************************************
-* Tuning parameters
-****************************************************************/
-/*!MEMORY_USAGE :
-* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
-* Increasing memory usage improves compression ratio
-* Reduced memory usage can improve speed, due to cache effect
-* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
-#ifndef FSE_MAX_MEMORY_USAGE
-# define FSE_MAX_MEMORY_USAGE 14
-#endif
-#ifndef FSE_DEFAULT_MEMORY_USAGE
-# define FSE_DEFAULT_MEMORY_USAGE 13
-#endif
-#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE)
-# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE"
-#endif
-
-/*!FSE_MAX_SYMBOL_VALUE :
-* Maximum symbol value authorized.
-* Required for proper stack allocation */
-#ifndef FSE_MAX_SYMBOL_VALUE
-# define FSE_MAX_SYMBOL_VALUE 255
-#endif
-
-/* **************************************************************
-* template functions type & suffix
-****************************************************************/
-#define FSE_FUNCTION_TYPE BYTE
-#define FSE_FUNCTION_EXTENSION
-#define FSE_DECODE_TYPE FSE_decode_t
-
-
-#endif /* !FSE_COMMONDEFS_ONLY */
-
-
-/* ***************************************************************
-* Constants
-*****************************************************************/
-#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
-#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
-#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
-#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
-#define FSE_MIN_TABLELOG 5
-
-#define FSE_TABLELOG_ABSOLUTE_MAX 15
-#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
-# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
-#endif
-
-#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3)
-
-
-#endif /* FSE_STATIC_LINKING_ONLY */
-
-
-#if defined (__cplusplus)
-}
-#endif
stage1/zstd/lib/common/fse_decompress.c
@@ -1,403 +0,0 @@
-/* ******************************************************************
- * FSE : Finite State Entropy decoder
- * Copyright (c) Yann Collet, Facebook, Inc.
- *
- * You can contact the author at :
- * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
- * - Public forum : https://groups.google.com/forum/#!forum/lz4c
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
-****************************************************************** */
-
-
-/* **************************************************************
-* Includes
-****************************************************************/
-#include "debug.h" /* assert */
-#include "bitstream.h"
-#include "compiler.h"
-#define FSE_STATIC_LINKING_ONLY
-#include "fse.h"
-#include "error_private.h"
-#define ZSTD_DEPS_NEED_MALLOC
-#include "zstd_deps.h"
-
-
-/* **************************************************************
-* Error Management
-****************************************************************/
-#define FSE_isError ERR_isError
-#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
-
-
-/* **************************************************************
-* Templates
-****************************************************************/
-/*
- designed to be included
- for type-specific functions (template emulation in C)
- Objective is to write these functions only once, for improved maintenance
-*/
-
-/* safety checks */
-#ifndef FSE_FUNCTION_EXTENSION
-# error "FSE_FUNCTION_EXTENSION must be defined"
-#endif
-#ifndef FSE_FUNCTION_TYPE
-# error "FSE_FUNCTION_TYPE must be defined"
-#endif
-
-/* Function names */
-#define FSE_CAT(X,Y) X##Y
-#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
-#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
-
-
-/* Function templates */
-FSE_DTable* FSE_createDTable (unsigned tableLog)
-{
- if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
- return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
-}
-
-void FSE_freeDTable (FSE_DTable* dt)
-{
- ZSTD_free(dt);
-}
-
-static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
-{
- void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
- FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
- U16* symbolNext = (U16*)workSpace;
- BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1);
-
- U32 const maxSV1 = maxSymbolValue + 1;
- U32 const tableSize = 1 << tableLog;
- U32 highThreshold = tableSize-1;
-
- /* Sanity Checks */
- if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge);
- if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
- if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
-
- /* Init, lay down lowprob symbols */
- { FSE_DTableHeader DTableH;
- DTableH.tableLog = (U16)tableLog;
- DTableH.fastMode = 1;
- { S16 const largeLimit= (S16)(1 << (tableLog-1));
- U32 s;
- for (s=0; s<maxSV1; s++) {
- if (normalizedCounter[s]==-1) {
- tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
- symbolNext[s] = 1;
- } else {
- if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
- symbolNext[s] = normalizedCounter[s];
- } } }
- ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
- }
-
- /* Spread symbols */
- if (highThreshold == tableSize - 1) {
- size_t const tableMask = tableSize-1;
- size_t const step = FSE_TABLESTEP(tableSize);
- /* First lay down the symbols in order.
- * We use a uint64_t to lay down 8 bytes at a time. This reduces branch
- * misses since small blocks generally have small table logs, so nearly
- * all symbols have counts <= 8. We ensure we have 8 bytes at the end of
- * our buffer to handle the over-write.
- */
- {
- U64 const add = 0x0101010101010101ull;
- size_t pos = 0;
- U64 sv = 0;
- U32 s;
- for (s=0; s<maxSV1; ++s, sv += add) {
- int i;
- int const n = normalizedCounter[s];
- MEM_write64(spread + pos, sv);
- for (i = 8; i < n; i += 8) {
- MEM_write64(spread + pos + i, sv);
- }
- pos += n;
- }
- }
- /* Now we spread those positions across the table.
- * The benefit of doing it in two stages is that we avoid the the
- * variable size inner loop, which caused lots of branch misses.
- * Now we can run through all the positions without any branch misses.
- * We unroll the loop twice, since that is what emperically worked best.
- */
- {
- size_t position = 0;
- size_t s;
- size_t const unroll = 2;
- assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
- for (s = 0; s < (size_t)tableSize; s += unroll) {
- size_t u;
- for (u = 0; u < unroll; ++u) {
- size_t const uPosition = (position + (u * step)) & tableMask;
- tableDecode[uPosition].symbol = spread[s + u];
- }
- position = (position + (unroll * step)) & tableMask;
- }
- assert(position == 0);
- }
- } else {
- U32 const tableMask = tableSize-1;
- U32 const step = FSE_TABLESTEP(tableSize);
- U32 s, position = 0;
- for (s=0; s<maxSV1; s++) {
- int i;
- for (i=0; i<normalizedCounter[s]; i++) {
- tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
- position = (position + step) & tableMask;
- while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
- } }
- if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
- }
-
- /* Build Decoding table */
- { U32 u;
- for (u=0; u<tableSize; u++) {
- FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
- U32 const nextState = symbolNext[symbol]++;
- tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
- tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
- } }
-
- return 0;
-}
-
-size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
-{
- return FSE_buildDTable_internal(dt, normalizedCounter, maxSymbolValue, tableLog, workSpace, wkspSize);
-}
-
-
-#ifndef FSE_COMMONDEFS_ONLY
-
-/*-*******************************************************
-* Decompression (Byte symbols)
-*********************************************************/
-size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
-{
- void* ptr = dt;
- FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
- void* dPtr = dt + 1;
- FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
-
- DTableH->tableLog = 0;
- DTableH->fastMode = 0;
-
- cell->newState = 0;
- cell->symbol = symbolValue;
- cell->nbBits = 0;
-
- return 0;
-}
-
-
-size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
-{
- void* ptr = dt;
- FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
- void* dPtr = dt + 1;
- FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
- const unsigned tableSize = 1 << nbBits;
- const unsigned tableMask = tableSize - 1;
- const unsigned maxSV1 = tableMask+1;
- unsigned s;
-
- /* Sanity checks */
- if (nbBits < 1) return ERROR(GENERIC); /* min size */
-
- /* Build Decoding Table */
- DTableH->tableLog = (U16)nbBits;
- DTableH->fastMode = 1;
- for (s=0; s<maxSV1; s++) {
- dinfo[s].newState = 0;
- dinfo[s].symbol = (BYTE)s;
- dinfo[s].nbBits = (BYTE)nbBits;
- }
-
- return 0;
-}
-
-FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
- void* dst, size_t maxDstSize,
- const void* cSrc, size_t cSrcSize,
- const FSE_DTable* dt, const unsigned fast)
-{
- BYTE* const ostart = (BYTE*) dst;
- BYTE* op = ostart;
- BYTE* const omax = op + maxDstSize;
- BYTE* const olimit = omax-3;
-
- BIT_DStream_t bitD;
- FSE_DState_t state1;
- FSE_DState_t state2;
-
- /* Init */
- CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
-
- FSE_initDState(&state1, &bitD, dt);
- FSE_initDState(&state2, &bitD, dt);
-
-#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
-
- /* 4 symbols per loop */
- for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) {
- op[0] = FSE_GETSYMBOL(&state1);
-
- if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
- BIT_reloadDStream(&bitD);
-
- op[1] = FSE_GETSYMBOL(&state2);
-
- if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
- { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
-
- op[2] = FSE_GETSYMBOL(&state1);
-
- if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
- BIT_reloadDStream(&bitD);
-
- op[3] = FSE_GETSYMBOL(&state2);
- }
-
- /* tail */
- /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
- while (1) {
- if (op>(omax-2)) return ERROR(dstSize_tooSmall);
- *op++ = FSE_GETSYMBOL(&state1);
- if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
- *op++ = FSE_GETSYMBOL(&state2);
- break;
- }
-
- if (op>(omax-2)) return ERROR(dstSize_tooSmall);
- *op++ = FSE_GETSYMBOL(&state2);
- if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
- *op++ = FSE_GETSYMBOL(&state1);
- break;
- } }
-
- return op-ostart;
-}
-
-
-size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
- const void* cSrc, size_t cSrcSize,
- const FSE_DTable* dt)
-{
- const void* ptr = dt;
- const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
- const U32 fastMode = DTableH->fastMode;
-
- /* select fast mode (static) */
- if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
- return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
-}
-
-
-size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
-{
- return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
-typedef struct {
- short ncount[FSE_MAX_SYMBOL_VALUE + 1];
- FSE_DTable dtable[1]; /* Dynamically sized */
-} FSE_DecompressWksp;
-
-
-FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
- void* dst, size_t dstCapacity,
- const void* cSrc, size_t cSrcSize,
- unsigned maxLog, void* workSpace, size_t wkspSize,
- int bmi2)
-{
- const BYTE* const istart = (const BYTE*)cSrc;
- const BYTE* ip = istart;
- unsigned tableLog;
- unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
- FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace;
-
- DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0);
- if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC);
-
- /* normal FSE decoding mode */
- {
- size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2);
- if (FSE_isError(NCountLength)) return NCountLength;
- if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
- assert(NCountLength <= cSrcSize);
- ip += NCountLength;
- cSrcSize -= NCountLength;
- }
-
- if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
- workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog);
- wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
-
- CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
-
- {
- const void* ptr = wksp->dtable;
- const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
- const U32 fastMode = DTableH->fastMode;
-
- /* select fast mode (static) */
- if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1);
- return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0);
- }
-}
-
-/* Avoids the FORCE_INLINE of the _body() function. */
-static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
-{
- return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0);
-}
-
-#if DYNAMIC_BMI2
-BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
-{
- return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1);
-}
-#endif
-
-size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2)
-{
-#if DYNAMIC_BMI2
- if (bmi2) {
- return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
- }
-#endif
- (void)bmi2;
- return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
-}
-
-
-typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
-
-#ifndef ZSTD_NO_UNUSED_FUNCTIONS
-size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) {
- U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)];
- return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp));
-}
-
-size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
-{
- /* Static analyzer seems unable to understand this table will be properly initialized later */
- U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
- return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp));
-}
-#endif
-
-
-#endif /* FSE_COMMONDEFS_ONLY */
stage1/zstd/lib/common/huf.h
@@ -1,364 +0,0 @@
-/* ******************************************************************
- * huff0 huffman codec,
- * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
- *
- * You can contact the author at :
- * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
-****************************************************************** */
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-#ifndef HUF_H_298734234
-#define HUF_H_298734234
-
-/* *** Dependencies *** */
-#include "zstd_deps.h" /* size_t */
-
-
-/* *** library symbols visibility *** */
-/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual,
- * HUF symbols remain "private" (internal symbols for library only).
- * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */
-#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
-# define HUF_PUBLIC_API __attribute__ ((visibility ("default")))
-#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
-# define HUF_PUBLIC_API __declspec(dllexport)
-#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
-# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */
-#else
-# define HUF_PUBLIC_API
-#endif
-
-
-/* ========================== */
-/* *** simple functions *** */
-/* ========================== */
-
-/** HUF_compress() :
- * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
- * 'dst' buffer must be already allocated.
- * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
- * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
- * @return : size of compressed data (<= `dstCapacity`).
- * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
- * if HUF_isError(return), compression failed (more details using HUF_getErrorName())
- */
-HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
- const void* src, size_t srcSize);
-
-/** HUF_decompress() :
- * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
- * into already allocated buffer 'dst', of minimum size 'dstSize'.
- * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
- * Note : in contrast with FSE, HUF_decompress can regenerate
- * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
- * because it knows size to regenerate (originalSize).
- * @return : size of regenerated data (== originalSize),
- * or an error code, which can be tested using HUF_isError()
- */
-HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,
- const void* cSrc, size_t cSrcSize);
-
-
-/* *** Tool functions *** */
-#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
-HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
-
-/* Error Management */
-HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
-HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
-
-
-/* *** Advanced function *** */
-
-/** HUF_compress2() :
- * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
- * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
- * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
-HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- unsigned maxSymbolValue, unsigned tableLog);
-
-/** HUF_compress4X_wksp() :
- * Same as HUF_compress2(), but uses externally allocated `workSpace`.
- * `workspace` must be at least as large as HUF_WORKSPACE_SIZE */
-#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
-#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
-HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- unsigned maxSymbolValue, unsigned tableLog,
- void* workSpace, size_t wkspSize);
-
-#endif /* HUF_H_298734234 */
-
-/* ******************************************************************
- * WARNING !!
- * The following section contains advanced and experimental definitions
- * which shall never be used in the context of a dynamic library,
- * because they are not guaranteed to remain stable in the future.
- * Only consider them in association with static linking.
- * *****************************************************************/
-#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
-#define HUF_H_HUF_STATIC_LINKING_ONLY
-
-/* *** Dependencies *** */
-#include "mem.h" /* U32 */
-#define FSE_STATIC_LINKING_ONLY
-#include "fse.h"
-
-
-/* *** Constants *** */
-#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
-#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
-#define HUF_SYMBOLVALUE_MAX 255
-
-#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
-#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
-# error "HUF_TABLELOG_MAX is too large !"
-#endif
-
-
-/* ****************************************
-* Static allocation
-******************************************/
-/* HUF buffer bounds */
-#define HUF_CTABLEBOUND 129
-#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */
-#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
-
-/* static allocation of HUF's Compression Table */
-/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */
-typedef size_t HUF_CElt; /* consider it an incomplete type */
-#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */
-#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t))
-#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
- HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */
-
-/* static allocation of HUF's DTable */
-typedef U32 HUF_DTable;
-#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
-#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
- HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
-#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
- HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
-
-
-/* ****************************************
-* Advanced decompression functions
-******************************************/
-size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
-#endif
-
-size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
-size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
-size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
-size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
-size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
-size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
-#endif
-
-
-/* ****************************************
- * HUF detailed API
- * ****************************************/
-
-/*! HUF_compress() does the following:
- * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
- * 2. (optional) refine tableLog using HUF_optimalTableLog()
- * 3. build Huffman table from count using HUF_buildCTable()
- * 4. save Huffman table to memory buffer using HUF_writeCTable()
- * 5. encode the data stream using HUF_compress4X_usingCTable()
- *
- * The following API allows targeting specific sub-functions for advanced tasks.
- * For example, it's possible to compress several blocks using the same 'CTable',
- * or to save and regenerate 'CTable' using external methods.
- */
-unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
-size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
-size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
-size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
-size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
-size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
-size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
-int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
-
-typedef enum {
- HUF_repeat_none, /**< Cannot use the previous table */
- HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
- HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
- } HUF_repeat;
-/** HUF_compress4X_repeat() :
- * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
- * If it uses hufTable it does not modify hufTable or repeat.
- * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
- * If preferRepeat then the old table will always be used if valid.
- * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
-size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
- const void* src, size_t srcSize,
- unsigned maxSymbolValue, unsigned tableLog,
- void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
- HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
-
-/** HUF_buildCTable_wksp() :
- * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
- * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
- */
-#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
-#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
-size_t HUF_buildCTable_wksp (HUF_CElt* tree,
- const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
- void* workSpace, size_t wkspSize);
-
-/*! HUF_readStats() :
- * Read compact Huffman tree, saved by HUF_writeCTable().
- * `huffWeight` is destination buffer.
- * @return : size read from `src` , or an error Code .
- * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
-size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
- U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
- const void* src, size_t srcSize);
-
-/*! HUF_readStats_wksp() :
- * Same as HUF_readStats() but takes an external workspace which must be
- * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE.
- * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
- */
-#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1)
-#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned))
-size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
- U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
- const void* src, size_t srcSize,
- void* workspace, size_t wkspSize,
- int bmi2);
-
-/** HUF_readCTable() :
- * Loading a CTable saved with HUF_writeCTable() */
-size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights);
-
-/** HUF_getNbBitsFromCTable() :
- * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
- * Note 1 : is not inlined, as HUF_CElt definition is private */
-U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue);
-
-/*
- * HUF_decompress() does the following:
- * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
- * 2. build Huffman table from save, using HUF_readDTableX?()
- * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
- */
-
-/** HUF_selectDecoder() :
- * Tells which decoder is likely to decode faster,
- * based on a set of pre-computed metrics.
- * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
- * Assumption : 0 < dstSize <= 128 KB */
-U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
-
-/**
- * The minimum workspace size for the `workSpace` used in
- * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
- *
- * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
- * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
- * Buffer overflow errors may potentially occur if code modifications result in
- * a required workspace size greater than that specified in the following
- * macro.
- */
-#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
-#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
-
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
-size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
-size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
-#endif
-
-size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-
-
-/* ====================== */
-/* single stream variants */
-/* ====================== */
-
-size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 */
-size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
-size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
-/** HUF_compress1X_repeat() :
- * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
- * If it uses hufTable it does not modify hufTable or repeat.
- * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
- * If preferRepeat then the old table will always be used if valid.
- * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
-size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
- const void* src, size_t srcSize,
- unsigned maxSymbolValue, unsigned tableLog,
- void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
- HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
-
-size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
-#endif
-
-size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
-size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
-size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
-size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
-#endif
-
-size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-
-/* BMI2 variants.
- * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
- */
-size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
-#endif
-size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
-size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
-#endif
-
-#endif /* HUF_STATIC_LINKING_ONLY */
-
-#if defined (__cplusplus)
-}
-#endif
stage1/zstd/lib/common/mem.h
@@ -1,442 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef MEM_H_MODULE
-#define MEM_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-/*-****************************************
-* Dependencies
-******************************************/
-#include <stddef.h> /* size_t, ptrdiff_t */
-#include "compiler.h" /* __has_builtin */
-#include "debug.h" /* DEBUG_STATIC_ASSERT */
-#include "zstd_deps.h" /* ZSTD_memcpy */
-
-
-/*-****************************************
-* Compiler specifics
-******************************************/
-#if defined(_MSC_VER) /* Visual Studio */
-# include <stdlib.h> /* _byteswap_ulong */
-# include <intrin.h> /* _byteswap_* */
-#endif
-#if defined(__GNUC__)
-# define MEM_STATIC static __inline __attribute__((unused))
-#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define MEM_STATIC static inline
-#elif defined(_MSC_VER)
-# define MEM_STATIC static __inline
-#else
-# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
-#endif
-
-/*-**************************************************************
-* Basic Types
-*****************************************************************/
-#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# if defined(_AIX)
-# include <inttypes.h>
-# else
-# include <stdint.h> /* intptr_t */
-# endif
- typedef uint8_t BYTE;
- typedef uint8_t U8;
- typedef int8_t S8;
- typedef uint16_t U16;
- typedef int16_t S16;
- typedef uint32_t U32;
- typedef int32_t S32;
- typedef uint64_t U64;
- typedef int64_t S64;
-#else
-# include <limits.h>
-#if CHAR_BIT != 8
-# error "this implementation requires char to be exactly 8-bit type"
-#endif
- typedef unsigned char BYTE;
- typedef unsigned char U8;
- typedef signed char S8;
-#if USHRT_MAX != 65535
-# error "this implementation requires short to be exactly 16-bit type"
-#endif
- typedef unsigned short U16;
- typedef signed short S16;
-#if UINT_MAX != 4294967295
-# error "this implementation requires int to be exactly 32-bit type"
-#endif
- typedef unsigned int U32;
- typedef signed int S32;
-/* note : there are no limits defined for long long type in C90.
- * limits exist in C99, however, in such case, <stdint.h> is preferred */
- typedef unsigned long long U64;
- typedef signed long long S64;
-#endif
-
-
-/*-**************************************************************
-* Memory I/O API
-*****************************************************************/
-/*=== Static platform detection ===*/
-MEM_STATIC unsigned MEM_32bits(void);
-MEM_STATIC unsigned MEM_64bits(void);
-MEM_STATIC unsigned MEM_isLittleEndian(void);
-
-/*=== Native unaligned read/write ===*/
-MEM_STATIC U16 MEM_read16(const void* memPtr);
-MEM_STATIC U32 MEM_read32(const void* memPtr);
-MEM_STATIC U64 MEM_read64(const void* memPtr);
-MEM_STATIC size_t MEM_readST(const void* memPtr);
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value);
-MEM_STATIC void MEM_write32(void* memPtr, U32 value);
-MEM_STATIC void MEM_write64(void* memPtr, U64 value);
-
-/*=== Little endian unaligned read/write ===*/
-MEM_STATIC U16 MEM_readLE16(const void* memPtr);
-MEM_STATIC U32 MEM_readLE24(const void* memPtr);
-MEM_STATIC U32 MEM_readLE32(const void* memPtr);
-MEM_STATIC U64 MEM_readLE64(const void* memPtr);
-MEM_STATIC size_t MEM_readLEST(const void* memPtr);
-
-MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val);
-MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val);
-MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32);
-MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64);
-MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val);
-
-/*=== Big endian unaligned read/write ===*/
-MEM_STATIC U32 MEM_readBE32(const void* memPtr);
-MEM_STATIC U64 MEM_readBE64(const void* memPtr);
-MEM_STATIC size_t MEM_readBEST(const void* memPtr);
-
-MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32);
-MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64);
-MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val);
-
-/*=== Byteswap ===*/
-MEM_STATIC U32 MEM_swap32(U32 in);
-MEM_STATIC U64 MEM_swap64(U64 in);
-MEM_STATIC size_t MEM_swapST(size_t in);
-
-
-/*-**************************************************************
-* Memory I/O Implementation
-*****************************************************************/
-/* MEM_FORCE_MEMORY_ACCESS :
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
- * The below switch allow to select different access method for improved performance.
- * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).
- * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
- * Method 2 : direct access. This method is portable but violate C standard.
- * It can generate buggy code on targets depending on alignment.
- * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
- * Prefer these methods in priority order (0 > 1 > 2)
- */
-#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
-# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
-# define MEM_FORCE_MEMORY_ACCESS 1
-# endif
-#endif
-
-MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
-MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
-
-MEM_STATIC unsigned MEM_isLittleEndian(void)
-{
-#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
- return 1;
-#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
- return 0;
-#elif defined(__clang__) && __LITTLE_ENDIAN__
- return 1;
-#elif defined(__clang__) && __BIG_ENDIAN__
- return 0;
-#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86)
- return 1;
-#elif defined(__DMC__) && defined(_M_IX86)
- return 1;
-#else
- const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
- return one.c[0];
-#endif
-}
-
-#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
-
-/* violates C standard, by lying on structure alignment.
-Only use if no other choice to achieve best performance on target platform */
-MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
-MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
-MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
-
-#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
-
-/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
-/* currently only defined for gcc and icc */
-#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
- __pragma( pack(push, 1) )
- typedef struct { U16 v; } unalign16;
- typedef struct { U32 v; } unalign32;
- typedef struct { U64 v; } unalign64;
- typedef struct { size_t v; } unalignArch;
- __pragma( pack(pop) )
-#else
- typedef struct { U16 v; } __attribute__((packed)) unalign16;
- typedef struct { U32 v; } __attribute__((packed)) unalign32;
- typedef struct { U64 v; } __attribute__((packed)) unalign64;
- typedef struct { size_t v; } __attribute__((packed)) unalignArch;
-#endif
-
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
-MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }
-
-#else
-
-/* default method, safe and standard.
- can sometimes prove slower */
-
-MEM_STATIC U16 MEM_read16(const void* memPtr)
-{
- U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
-}
-
-MEM_STATIC U32 MEM_read32(const void* memPtr)
-{
- U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
-}
-
-MEM_STATIC U64 MEM_read64(const void* memPtr)
-{
- U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
-}
-
-MEM_STATIC size_t MEM_readST(const void* memPtr)
-{
- size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
-}
-
-MEM_STATIC void MEM_write16(void* memPtr, U16 value)
-{
- ZSTD_memcpy(memPtr, &value, sizeof(value));
-}
-
-MEM_STATIC void MEM_write32(void* memPtr, U32 value)
-{
- ZSTD_memcpy(memPtr, &value, sizeof(value));
-}
-
-MEM_STATIC void MEM_write64(void* memPtr, U64 value)
-{
- ZSTD_memcpy(memPtr, &value, sizeof(value));
-}
-
-#endif /* MEM_FORCE_MEMORY_ACCESS */
-
-MEM_STATIC U32 MEM_swap32(U32 in)
-{
-#if defined(_MSC_VER) /* Visual Studio */
- return _byteswap_ulong(in);
-#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
- || (defined(__clang__) && __has_builtin(__builtin_bswap32))
- return __builtin_bswap32(in);
-#else
- return ((in << 24) & 0xff000000 ) |
- ((in << 8) & 0x00ff0000 ) |
- ((in >> 8) & 0x0000ff00 ) |
- ((in >> 24) & 0x000000ff );
-#endif
-}
-
-MEM_STATIC U64 MEM_swap64(U64 in)
-{
-#if defined(_MSC_VER) /* Visual Studio */
- return _byteswap_uint64(in);
-#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
- || (defined(__clang__) && __has_builtin(__builtin_bswap64))
- return __builtin_bswap64(in);
-#else
- return ((in << 56) & 0xff00000000000000ULL) |
- ((in << 40) & 0x00ff000000000000ULL) |
- ((in << 24) & 0x0000ff0000000000ULL) |
- ((in << 8) & 0x000000ff00000000ULL) |
- ((in >> 8) & 0x00000000ff000000ULL) |
- ((in >> 24) & 0x0000000000ff0000ULL) |
- ((in >> 40) & 0x000000000000ff00ULL) |
- ((in >> 56) & 0x00000000000000ffULL);
-#endif
-}
-
-MEM_STATIC size_t MEM_swapST(size_t in)
-{
- if (MEM_32bits())
- return (size_t)MEM_swap32((U32)in);
- else
- return (size_t)MEM_swap64((U64)in);
-}
-
-/*=== Little endian r/w ===*/
-
-MEM_STATIC U16 MEM_readLE16(const void* memPtr)
-{
- if (MEM_isLittleEndian())
- return MEM_read16(memPtr);
- else {
- const BYTE* p = (const BYTE*)memPtr;
- return (U16)(p[0] + (p[1]<<8));
- }
-}
-
-MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
-{
- if (MEM_isLittleEndian()) {
- MEM_write16(memPtr, val);
- } else {
- BYTE* p = (BYTE*)memPtr;
- p[0] = (BYTE)val;
- p[1] = (BYTE)(val>>8);
- }
-}
-
-MEM_STATIC U32 MEM_readLE24(const void* memPtr)
-{
- return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16);
-}
-
-MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
-{
- MEM_writeLE16(memPtr, (U16)val);
- ((BYTE*)memPtr)[2] = (BYTE)(val>>16);
-}
-
-MEM_STATIC U32 MEM_readLE32(const void* memPtr)
-{
- if (MEM_isLittleEndian())
- return MEM_read32(memPtr);
- else
- return MEM_swap32(MEM_read32(memPtr));
-}
-
-MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
-{
- if (MEM_isLittleEndian())
- MEM_write32(memPtr, val32);
- else
- MEM_write32(memPtr, MEM_swap32(val32));
-}
-
-MEM_STATIC U64 MEM_readLE64(const void* memPtr)
-{
- if (MEM_isLittleEndian())
- return MEM_read64(memPtr);
- else
- return MEM_swap64(MEM_read64(memPtr));
-}
-
-MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
-{
- if (MEM_isLittleEndian())
- MEM_write64(memPtr, val64);
- else
- MEM_write64(memPtr, MEM_swap64(val64));
-}
-
-MEM_STATIC size_t MEM_readLEST(const void* memPtr)
-{
- if (MEM_32bits())
- return (size_t)MEM_readLE32(memPtr);
- else
- return (size_t)MEM_readLE64(memPtr);
-}
-
-MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
-{
- if (MEM_32bits())
- MEM_writeLE32(memPtr, (U32)val);
- else
- MEM_writeLE64(memPtr, (U64)val);
-}
-
-/*=== Big endian r/w ===*/
-
-MEM_STATIC U32 MEM_readBE32(const void* memPtr)
-{
- if (MEM_isLittleEndian())
- return MEM_swap32(MEM_read32(memPtr));
- else
- return MEM_read32(memPtr);
-}
-
-MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
-{
- if (MEM_isLittleEndian())
- MEM_write32(memPtr, MEM_swap32(val32));
- else
- MEM_write32(memPtr, val32);
-}
-
-MEM_STATIC U64 MEM_readBE64(const void* memPtr)
-{
- if (MEM_isLittleEndian())
- return MEM_swap64(MEM_read64(memPtr));
- else
- return MEM_read64(memPtr);
-}
-
-MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
-{
- if (MEM_isLittleEndian())
- MEM_write64(memPtr, MEM_swap64(val64));
- else
- MEM_write64(memPtr, val64);
-}
-
-MEM_STATIC size_t MEM_readBEST(const void* memPtr)
-{
- if (MEM_32bits())
- return (size_t)MEM_readBE32(memPtr);
- else
- return (size_t)MEM_readBE64(memPtr);
-}
-
-MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
-{
- if (MEM_32bits())
- MEM_writeBE32(memPtr, (U32)val);
- else
- MEM_writeBE64(memPtr, (U64)val);
-}
-
-/* code only tested on 32 and 64 bits systems */
-MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* MEM_H_MODULE */
stage1/zstd/lib/common/pool.c
@@ -1,355 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-
-/* ====== Dependencies ======= */
-#include "zstd_deps.h" /* size_t */
-#include "debug.h" /* assert */
-#include "zstd_internal.h" /* ZSTD_customMalloc, ZSTD_customFree */
-#include "pool.h"
-
-/* ====== Compiler specifics ====== */
-#if defined(_MSC_VER)
-# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
-#endif
-
-
-#ifdef ZSTD_MULTITHREAD
-
-#include "threading.h" /* pthread adaptation */
-
-/* A job is a function and an opaque argument */
-typedef struct POOL_job_s {
- POOL_function function;
- void *opaque;
-} POOL_job;
-
-struct POOL_ctx_s {
- ZSTD_customMem customMem;
- /* Keep track of the threads */
- ZSTD_pthread_t* threads;
- size_t threadCapacity;
- size_t threadLimit;
-
- /* The queue is a circular buffer */
- POOL_job *queue;
- size_t queueHead;
- size_t queueTail;
- size_t queueSize;
-
- /* The number of threads working on jobs */
- size_t numThreadsBusy;
- /* Indicates if the queue is empty */
- int queueEmpty;
-
- /* The mutex protects the queue */
- ZSTD_pthread_mutex_t queueMutex;
- /* Condition variable for pushers to wait on when the queue is full */
- ZSTD_pthread_cond_t queuePushCond;
- /* Condition variables for poppers to wait on when the queue is empty */
- ZSTD_pthread_cond_t queuePopCond;
- /* Indicates if the queue is shutting down */
- int shutdown;
-};
-
-/* POOL_thread() :
- * Work thread for the thread pool.
- * Waits for jobs and executes them.
- * @returns : NULL on failure else non-null.
- */
-static void* POOL_thread(void* opaque) {
- POOL_ctx* const ctx = (POOL_ctx*)opaque;
- if (!ctx) { return NULL; }
- for (;;) {
- /* Lock the mutex and wait for a non-empty queue or until shutdown */
- ZSTD_pthread_mutex_lock(&ctx->queueMutex);
-
- while ( ctx->queueEmpty
- || (ctx->numThreadsBusy >= ctx->threadLimit) ) {
- if (ctx->shutdown) {
- /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit),
- * a few threads will be shutdown while !queueEmpty,
- * but enough threads will remain active to finish the queue */
- ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
- return opaque;
- }
- ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
- }
- /* Pop a job off the queue */
- { POOL_job const job = ctx->queue[ctx->queueHead];
- ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
- ctx->numThreadsBusy++;
- ctx->queueEmpty = (ctx->queueHead == ctx->queueTail);
- /* Unlock the mutex, signal a pusher, and run the job */
- ZSTD_pthread_cond_signal(&ctx->queuePushCond);
- ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
-
- job.function(job.opaque);
-
- /* If the intended queue size was 0, signal after finishing job */
- ZSTD_pthread_mutex_lock(&ctx->queueMutex);
- ctx->numThreadsBusy--;
- if (ctx->queueSize == 1) {
- ZSTD_pthread_cond_signal(&ctx->queuePushCond);
- }
- ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
- }
- } /* for (;;) */
- assert(0); /* Unreachable */
-}
-
-/* ZSTD_createThreadPool() : public access point */
-POOL_ctx* ZSTD_createThreadPool(size_t numThreads) {
- return POOL_create (numThreads, 0);
-}
-
-POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
- return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
-}
-
-POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
- ZSTD_customMem customMem)
-{
- POOL_ctx* ctx;
- /* Check parameters */
- if (!numThreads) { return NULL; }
- /* Allocate the context and zero initialize */
- ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem);
- if (!ctx) { return NULL; }
- /* Initialize the job queue.
- * It needs one extra space since one space is wasted to differentiate
- * empty and full queues.
- */
- ctx->queueSize = queueSize + 1;
- ctx->queue = (POOL_job*)ZSTD_customMalloc(ctx->queueSize * sizeof(POOL_job), customMem);
- ctx->queueHead = 0;
- ctx->queueTail = 0;
- ctx->numThreadsBusy = 0;
- ctx->queueEmpty = 1;
- {
- int error = 0;
- error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
- error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
- error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
- if (error) { POOL_free(ctx); return NULL; }
- }
- ctx->shutdown = 0;
- /* Allocate space for the thread handles */
- ctx->threads = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
- ctx->threadCapacity = 0;
- ctx->customMem = customMem;
- /* Check for errors */
- if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
- /* Initialize the threads */
- { size_t i;
- for (i = 0; i < numThreads; ++i) {
- if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
- ctx->threadCapacity = i;
- POOL_free(ctx);
- return NULL;
- } }
- ctx->threadCapacity = numThreads;
- ctx->threadLimit = numThreads;
- }
- return ctx;
-}
-
-/*! POOL_join() :
- Shutdown the queue, wake any sleeping threads, and join all of the threads.
-*/
-static void POOL_join(POOL_ctx* ctx) {
- /* Shut down the queue */
- ZSTD_pthread_mutex_lock(&ctx->queueMutex);
- ctx->shutdown = 1;
- ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
- /* Wake up sleeping threads */
- ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
- ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
- /* Join all of the threads */
- { size_t i;
- for (i = 0; i < ctx->threadCapacity; ++i) {
- ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */
- } }
-}
-
-void POOL_free(POOL_ctx *ctx) {
- if (!ctx) { return; }
- POOL_join(ctx);
- ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
- ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
- ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
- ZSTD_customFree(ctx->queue, ctx->customMem);
- ZSTD_customFree(ctx->threads, ctx->customMem);
- ZSTD_customFree(ctx, ctx->customMem);
-}
-
-void ZSTD_freeThreadPool (ZSTD_threadPool* pool) {
- POOL_free (pool);
-}
-
-size_t POOL_sizeof(const POOL_ctx* ctx) {
- if (ctx==NULL) return 0; /* supports sizeof NULL */
- return sizeof(*ctx)
- + ctx->queueSize * sizeof(POOL_job)
- + ctx->threadCapacity * sizeof(ZSTD_pthread_t);
-}
-
-
-/* @return : 0 on success, 1 on error */
-static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
-{
- if (numThreads <= ctx->threadCapacity) {
- if (!numThreads) return 1;
- ctx->threadLimit = numThreads;
- return 0;
- }
- /* numThreads > threadCapacity */
- { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
- if (!threadPool) return 1;
- /* replace existing thread pool */
- ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
- ZSTD_customFree(ctx->threads, ctx->customMem);
- ctx->threads = threadPool;
- /* Initialize additional threads */
- { size_t threadId;
- for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
- if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
- ctx->threadCapacity = threadId;
- return 1;
- } }
- } }
- /* successfully expanded */
- ctx->threadCapacity = numThreads;
- ctx->threadLimit = numThreads;
- return 0;
-}
-
-/* @return : 0 on success, 1 on error */
-int POOL_resize(POOL_ctx* ctx, size_t numThreads)
-{
- int result;
- if (ctx==NULL) return 1;
- ZSTD_pthread_mutex_lock(&ctx->queueMutex);
- result = POOL_resize_internal(ctx, numThreads);
- ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
- ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
- return result;
-}
-
-/**
- * Returns 1 if the queue is full and 0 otherwise.
- *
- * When queueSize is 1 (pool was created with an intended queueSize of 0),
- * then a queue is empty if there is a thread free _and_ no job is waiting.
- */
-static int isQueueFull(POOL_ctx const* ctx) {
- if (ctx->queueSize > 1) {
- return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
- } else {
- return (ctx->numThreadsBusy == ctx->threadLimit) ||
- !ctx->queueEmpty;
- }
-}
-
-
-static void
-POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
-{
- POOL_job const job = {function, opaque};
- assert(ctx != NULL);
- if (ctx->shutdown) return;
-
- ctx->queueEmpty = 0;
- ctx->queue[ctx->queueTail] = job;
- ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
- ZSTD_pthread_cond_signal(&ctx->queuePopCond);
-}
-
-void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque)
-{
- assert(ctx != NULL);
- ZSTD_pthread_mutex_lock(&ctx->queueMutex);
- /* Wait until there is space in the queue for the new job */
- while (isQueueFull(ctx) && (!ctx->shutdown)) {
- ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
- }
- POOL_add_internal(ctx, function, opaque);
- ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
-}
-
-
-int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque)
-{
- assert(ctx != NULL);
- ZSTD_pthread_mutex_lock(&ctx->queueMutex);
- if (isQueueFull(ctx)) {
- ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
- return 0;
- }
- POOL_add_internal(ctx, function, opaque);
- ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
- return 1;
-}
-
-
-#else /* ZSTD_MULTITHREAD not defined */
-
-/* ========================== */
-/* No multi-threading support */
-/* ========================== */
-
-
-/* We don't need any data, but if it is empty, malloc() might return NULL. */
-struct POOL_ctx_s {
- int dummy;
-};
-static POOL_ctx g_poolCtx;
-
-POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
- return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
-}
-
-POOL_ctx*
-POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem)
-{
- (void)numThreads;
- (void)queueSize;
- (void)customMem;
- return &g_poolCtx;
-}
-
-void POOL_free(POOL_ctx* ctx) {
- assert(!ctx || ctx == &g_poolCtx);
- (void)ctx;
-}
-
-int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
- (void)ctx; (void)numThreads;
- return 0;
-}
-
-void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
- (void)ctx;
- function(opaque);
-}
-
-int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
- (void)ctx;
- function(opaque);
- return 1;
-}
-
-size_t POOL_sizeof(const POOL_ctx* ctx) {
- if (ctx==NULL) return 0; /* supports sizeof NULL */
- assert(ctx == &g_poolCtx);
- return sizeof(*ctx);
-}
-
-#endif /* ZSTD_MULTITHREAD */
stage1/zstd/lib/common/pool.h
@@ -1,84 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef POOL_H
-#define POOL_H
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-#include "zstd_deps.h"
-#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */
-#include "../zstd.h"
-
-typedef struct POOL_ctx_s POOL_ctx;
-
-/*! POOL_create() :
- * Create a thread pool with at most `numThreads` threads.
- * `numThreads` must be at least 1.
- * The maximum number of queued jobs before blocking is `queueSize`.
- * @return : POOL_ctx pointer on success, else NULL.
-*/
-POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
-
-POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
- ZSTD_customMem customMem);
-
-/*! POOL_free() :
- * Free a thread pool returned by POOL_create().
- */
-void POOL_free(POOL_ctx* ctx);
-
-/*! POOL_resize() :
- * Expands or shrinks pool's number of threads.
- * This is more efficient than releasing + creating a new context,
- * since it tries to preserve and re-use existing threads.
- * `numThreads` must be at least 1.
- * @return : 0 when resize was successful,
- * !0 (typically 1) if there is an error.
- * note : only numThreads can be resized, queueSize remains unchanged.
- */
-int POOL_resize(POOL_ctx* ctx, size_t numThreads);
-
-/*! POOL_sizeof() :
- * @return threadpool memory usage
- * note : compatible with NULL (returns 0 in this case)
- */
-size_t POOL_sizeof(const POOL_ctx* ctx);
-
-/*! POOL_function :
- * The function type that can be added to a thread pool.
- */
-typedef void (*POOL_function)(void*);
-
-/*! POOL_add() :
- * Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
- * Possibly blocks until there is room in the queue.
- * Note : The function may be executed asynchronously,
- * therefore, `opaque` must live until function has been completed.
- */
-void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
-
-
-/*! POOL_tryAdd() :
- * Add the job `function(opaque)` to thread pool _if_ a queue slot is available.
- * Returns immediately even if not (does not block).
- * @return : 1 if successful, 0 if not.
- */
-int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif
stage1/zstd/lib/common/portability_macros.h
@@ -1,137 +0,0 @@
-/*
- * Copyright (c) Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef ZSTD_PORTABILITY_MACROS_H
-#define ZSTD_PORTABILITY_MACROS_H
-
-/**
- * This header file contains macro defintions to support portability.
- * This header is shared between C and ASM code, so it MUST only
- * contain macro definitions. It MUST not contain any C code.
- *
- * This header ONLY defines macros to detect platforms/feature support.
- *
- */
-
-
-/* compat. with non-clang compilers */
-#ifndef __has_attribute
- #define __has_attribute(x) 0
-#endif
-
-/* compat. with non-clang compilers */
-#ifndef __has_builtin
-# define __has_builtin(x) 0
-#endif
-
-/* compat. with non-clang compilers */
-#ifndef __has_feature
-# define __has_feature(x) 0
-#endif
-
-/* detects whether we are being compiled under msan */
-#ifndef ZSTD_MEMORY_SANITIZER
-# if __has_feature(memory_sanitizer)
-# define ZSTD_MEMORY_SANITIZER 1
-# else
-# define ZSTD_MEMORY_SANITIZER 0
-# endif
-#endif
-
-/* detects whether we are being compiled under asan */
-#ifndef ZSTD_ADDRESS_SANITIZER
-# if __has_feature(address_sanitizer)
-# define ZSTD_ADDRESS_SANITIZER 1
-# elif defined(__SANITIZE_ADDRESS__)
-# define ZSTD_ADDRESS_SANITIZER 1
-# else
-# define ZSTD_ADDRESS_SANITIZER 0
-# endif
-#endif
-
-/* detects whether we are being compiled under dfsan */
-#ifndef ZSTD_DATAFLOW_SANITIZER
-# if __has_feature(dataflow_sanitizer)
-# define ZSTD_DATAFLOW_SANITIZER 1
-# else
-# define ZSTD_DATAFLOW_SANITIZER 0
-# endif
-#endif
-
-/* Mark the internal assembly functions as hidden */
-#ifdef __ELF__
-# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
-#else
-# define ZSTD_HIDE_ASM_FUNCTION(func)
-#endif
-
-/* Enable runtime BMI2 dispatch based on the CPU.
- * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
- */
-#ifndef DYNAMIC_BMI2
- #if ((defined(__clang__) && __has_attribute(__target__)) \
- || (defined(__GNUC__) \
- && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
- && (defined(__x86_64__) || defined(_M_X64)) \
- && !defined(__BMI2__)
- # define DYNAMIC_BMI2 1
- #else
- # define DYNAMIC_BMI2 0
- #endif
-#endif
-
-/**
- * Only enable assembly for GNUC comptabile compilers,
- * because other platforms may not support GAS assembly syntax.
- *
- * Only enable assembly for Linux / MacOS, other platforms may
- * work, but they haven't been tested. This could likely be
- * extended to BSD systems.
- *
- * Disable assembly when MSAN is enabled, because MSAN requires
- * 100% of code to be instrumented to work.
- */
-#if defined(__GNUC__)
-# if defined(__linux__) || defined(__linux) || defined(__APPLE__)
-# if ZSTD_MEMORY_SANITIZER
-# define ZSTD_ASM_SUPPORTED 0
-# elif ZSTD_DATAFLOW_SANITIZER
-# define ZSTD_ASM_SUPPORTED 0
-# else
-# define ZSTD_ASM_SUPPORTED 1
-# endif
-# else
-# define ZSTD_ASM_SUPPORTED 0
-# endif
-#else
-# define ZSTD_ASM_SUPPORTED 0
-#endif
-
-/**
- * Determines whether we should enable assembly for x86-64
- * with BMI2.
- *
- * Enable if all of the following conditions hold:
- * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM
- * - Assembly is supported
- * - We are compiling for x86-64 and either:
- * - DYNAMIC_BMI2 is enabled
- * - BMI2 is supported at compile time
- */
-#if !defined(ZSTD_DISABLE_ASM) && \
- ZSTD_ASM_SUPPORTED && \
- defined(__x86_64__) && \
- (DYNAMIC_BMI2 || defined(__BMI2__))
-# define ZSTD_ENABLE_ASM_X86_64_BMI2 1
-#else
-# define ZSTD_ENABLE_ASM_X86_64_BMI2 0
-#endif
-
-#endif /* ZSTD_PORTABILITY_MACROS_H */
stage1/zstd/lib/common/threading.c
@@ -1,122 +0,0 @@
-/**
- * Copyright (c) 2016 Tino Reichardt
- * All rights reserved.
- *
- * You can contact the author at:
- * - zstdmt source repository: https://github.com/mcmilk/zstdmt
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-/**
- * This file will hold wrapper for systems, which do not support pthreads
- */
-
-#include "threading.h"
-
-/* create fake symbol to avoid empty translation unit warning */
-int g_ZSTD_threading_useless_symbol;
-
-#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
-
-/**
- * Windows minimalist Pthread Wrapper, based on :
- * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
- */
-
-
-/* === Dependencies === */
-#include <process.h>
-#include <errno.h>
-
-
-/* === Implementation === */
-
-static unsigned __stdcall worker(void *arg)
-{
- ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
- thread->arg = thread->start_routine(thread->arg);
- return 0;
-}
-
-int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
- void* (*start_routine) (void*), void* arg)
-{
- (void)unused;
- thread->arg = arg;
- thread->start_routine = start_routine;
- thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
-
- if (!thread->handle)
- return errno;
- else
- return 0;
-}
-
-int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
-{
- DWORD result;
-
- if (!thread.handle) return 0;
-
- result = WaitForSingleObject(thread.handle, INFINITE);
- switch (result) {
- case WAIT_OBJECT_0:
- if (value_ptr) *value_ptr = thread.arg;
- return 0;
- case WAIT_ABANDONED:
- return EINVAL;
- default:
- return GetLastError();
- }
-}
-
-#endif /* ZSTD_MULTITHREAD */
-
-#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32)
-
-#define ZSTD_DEPS_NEED_MALLOC
-#include "zstd_deps.h"
-
-int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr)
-{
- *mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t));
- if (!*mutex)
- return 1;
- return pthread_mutex_init(*mutex, attr);
-}
-
-int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
-{
- if (!*mutex)
- return 0;
- {
- int const ret = pthread_mutex_destroy(*mutex);
- ZSTD_free(*mutex);
- return ret;
- }
-}
-
-int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr)
-{
- *cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t));
- if (!*cond)
- return 1;
- return pthread_cond_init(*cond, attr);
-}
-
-int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
-{
- if (!*cond)
- return 0;
- {
- int const ret = pthread_cond_destroy(*cond);
- ZSTD_free(*cond);
- return ret;
- }
-}
-
-#endif
stage1/zstd/lib/common/threading.h
@@ -1,155 +0,0 @@
-/**
- * Copyright (c) 2016 Tino Reichardt
- * All rights reserved.
- *
- * You can contact the author at:
- * - zstdmt source repository: https://github.com/mcmilk/zstdmt
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef THREADING_H_938743
-#define THREADING_H_938743
-
-#include "debug.h"
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
-
-/**
- * Windows minimalist Pthread Wrapper, based on :
- * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
- */
-#ifdef WINVER
-# undef WINVER
-#endif
-#define WINVER 0x0600
-
-#ifdef _WIN32_WINNT
-# undef _WIN32_WINNT
-#endif
-#define _WIN32_WINNT 0x0600
-
-#ifndef WIN32_LEAN_AND_MEAN
-# define WIN32_LEAN_AND_MEAN
-#endif
-
-#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
-#include <windows.h>
-#undef ERROR
-#define ERROR(name) ZSTD_ERROR(name)
-
-
-/* mutex */
-#define ZSTD_pthread_mutex_t CRITICAL_SECTION
-#define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0)
-#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a))
-#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a))
-#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a))
-
-/* condition variable */
-#define ZSTD_pthread_cond_t CONDITION_VARIABLE
-#define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0)
-#define ZSTD_pthread_cond_destroy(a) ((void)(a))
-#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE)
-#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a))
-#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a))
-
-/* ZSTD_pthread_create() and ZSTD_pthread_join() */
-typedef struct {
- HANDLE handle;
- void* (*start_routine)(void*);
- void* arg;
-} ZSTD_pthread_t;
-
-int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
- void* (*start_routine) (void*), void* arg);
-
-int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
-
-/**
- * add here more wrappers as required
- */
-
-
-#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */
-/* === POSIX Systems === */
-# include <pthread.h>
-
-#if DEBUGLEVEL < 1
-
-#define ZSTD_pthread_mutex_t pthread_mutex_t
-#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b))
-#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a))
-#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a))
-#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a))
-
-#define ZSTD_pthread_cond_t pthread_cond_t
-#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b))
-#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a))
-#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b))
-#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a))
-#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a))
-
-#define ZSTD_pthread_t pthread_t
-#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
-#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
-
-#else /* DEBUGLEVEL >= 1 */
-
-/* Debug implementation of threading.
- * In this implementation we use pointers for mutexes and condition variables.
- * This way, if we forget to init/destroy them the program will crash or ASAN
- * will report leaks.
- */
-
-#define ZSTD_pthread_mutex_t pthread_mutex_t*
-int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr);
-int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex);
-#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a))
-#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a))
-
-#define ZSTD_pthread_cond_t pthread_cond_t*
-int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr);
-int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
-#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b))
-#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a))
-#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a))
-
-#define ZSTD_pthread_t pthread_t
-#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
-#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
-
-#endif
-
-#else /* ZSTD_MULTITHREAD not defined */
-/* No multithreading support */
-
-typedef int ZSTD_pthread_mutex_t;
-#define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0)
-#define ZSTD_pthread_mutex_destroy(a) ((void)(a))
-#define ZSTD_pthread_mutex_lock(a) ((void)(a))
-#define ZSTD_pthread_mutex_unlock(a) ((void)(a))
-
-typedef int ZSTD_pthread_cond_t;
-#define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0)
-#define ZSTD_pthread_cond_destroy(a) ((void)(a))
-#define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b))
-#define ZSTD_pthread_cond_signal(a) ((void)(a))
-#define ZSTD_pthread_cond_broadcast(a) ((void)(a))
-
-/* do not use ZSTD_pthread_t */
-
-#endif /* ZSTD_MULTITHREAD */
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* THREADING_H_938743 */
stage1/zstd/lib/common/xxhash.c
@@ -1,24 +0,0 @@
-/*
- * xxHash - Fast Hash algorithm
- * Copyright (c) Yann Collet, Facebook, Inc.
- *
- * You can contact the author at :
- * - xxHash homepage: http://www.xxhash.com
- * - xxHash source repository : https://github.com/Cyan4973/xxHash
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
-*/
-
-
-
-/*
- * xxhash.c instantiates functions defined in xxhash.h
- */
-
-#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
-#define XXH_IMPLEMENTATION /* access definitions */
-
-#include "xxhash.h"
stage1/zstd/lib/common/xxhash.h
@@ -1,5686 +0,0 @@
-/*
- * xxHash - Fast Hash algorithm
- * Copyright (c) Yann Collet, Facebook, Inc.
- *
- * You can contact the author at :
- * - xxHash homepage: http://www.xxhash.com
- * - xxHash source repository : https://github.com/Cyan4973/xxHash
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
-*/
-
-
-#ifndef XXH_NO_XXH3
-# define XXH_NO_XXH3
-#endif
-
-#ifndef XXH_NAMESPACE
-# define XXH_NAMESPACE ZSTD_
-#endif
-
-/*!
- * @mainpage xxHash
- *
- * @file xxhash.h
- * xxHash prototypes and implementation
- */
-/* TODO: update */
-/* Notice extracted from xxHash homepage:
-
-xxHash is an extremely fast hash algorithm, running at RAM speed limits.
-It also successfully passes all tests from the SMHasher suite.
-
-Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
-
-Name Speed Q.Score Author
-xxHash 5.4 GB/s 10
-CrapWow 3.2 GB/s 2 Andrew
-MurmurHash 3a 2.7 GB/s 10 Austin Appleby
-SpookyHash 2.0 GB/s 10 Bob Jenkins
-SBox 1.4 GB/s 9 Bret Mulvey
-Lookup3 1.2 GB/s 9 Bob Jenkins
-SuperFastHash 1.2 GB/s 1 Paul Hsieh
-CityHash64 1.05 GB/s 10 Pike & Alakuijala
-FNV 0.55 GB/s 5 Fowler, Noll, Vo
-CRC32 0.43 GB/s 9
-MD5-32 0.33 GB/s 10 Ronald L. Rivest
-SHA1-32 0.28 GB/s 10
-
-Q.Score is a measure of quality of the hash function.
-It depends on successfully passing SMHasher test set.
-10 is a perfect score.
-
-Note: SMHasher's CRC32 implementation is not the fastest one.
-Other speed-oriented implementations can be faster,
-especially in combination with PCLMUL instruction:
-https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735
-
-A 64-bit version, named XXH64, is available since r35.
-It offers much better speed, but for 64-bit applications only.
-Name Speed on 64 bits Speed on 32 bits
-XXH64 13.8 GB/s 1.9 GB/s
-XXH32 6.8 GB/s 6.0 GB/s
-*/
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-/* ****************************
- * INLINE mode
- ******************************/
-/*!
- * XXH_INLINE_ALL (and XXH_PRIVATE_API)
- * Use these build macros to inline xxhash into the target unit.
- * Inlining improves performance on small inputs, especially when the length is
- * expressed as a compile-time constant:
- *
- * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html
- *
- * It also keeps xxHash symbols private to the unit, so they are not exported.
- *
- * Usage:
- * #define XXH_INLINE_ALL
- * #include "xxhash.h"
- *
- * Do not compile and link xxhash.o as a separate object, as it is not useful.
- */
-#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \
- && !defined(XXH_INLINE_ALL_31684351384)
- /* this section should be traversed only once */
-# define XXH_INLINE_ALL_31684351384
- /* give access to the advanced API, required to compile implementations */
-# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */
-# define XXH_STATIC_LINKING_ONLY
- /* make all functions private */
-# undef XXH_PUBLIC_API
-# if defined(__GNUC__)
-# define XXH_PUBLIC_API static __inline __attribute__((unused))
-# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define XXH_PUBLIC_API static inline
-# elif defined(_MSC_VER)
-# define XXH_PUBLIC_API static __inline
-# else
- /* note: this version may generate warnings for unused static functions */
-# define XXH_PUBLIC_API static
-# endif
-
- /*
- * This part deals with the special case where a unit wants to inline xxHash,
- * but "xxhash.h" has previously been included without XXH_INLINE_ALL,
- * such as part of some previously included *.h header file.
- * Without further action, the new include would just be ignored,
- * and functions would effectively _not_ be inlined (silent failure).
- * The following macros solve this situation by prefixing all inlined names,
- * avoiding naming collision with previous inclusions.
- */
- /* Before that, we unconditionally #undef all symbols,
- * in case they were already defined with XXH_NAMESPACE.
- * They will then be redefined for XXH_INLINE_ALL
- */
-# undef XXH_versionNumber
- /* XXH32 */
-# undef XXH32
-# undef XXH32_createState
-# undef XXH32_freeState
-# undef XXH32_reset
-# undef XXH32_update
-# undef XXH32_digest
-# undef XXH32_copyState
-# undef XXH32_canonicalFromHash
-# undef XXH32_hashFromCanonical
- /* XXH64 */
-# undef XXH64
-# undef XXH64_createState
-# undef XXH64_freeState
-# undef XXH64_reset
-# undef XXH64_update
-# undef XXH64_digest
-# undef XXH64_copyState
-# undef XXH64_canonicalFromHash
-# undef XXH64_hashFromCanonical
- /* XXH3_64bits */
-# undef XXH3_64bits
-# undef XXH3_64bits_withSecret
-# undef XXH3_64bits_withSeed
-# undef XXH3_64bits_withSecretandSeed
-# undef XXH3_createState
-# undef XXH3_freeState
-# undef XXH3_copyState
-# undef XXH3_64bits_reset
-# undef XXH3_64bits_reset_withSeed
-# undef XXH3_64bits_reset_withSecret
-# undef XXH3_64bits_update
-# undef XXH3_64bits_digest
-# undef XXH3_generateSecret
- /* XXH3_128bits */
-# undef XXH128
-# undef XXH3_128bits
-# undef XXH3_128bits_withSeed
-# undef XXH3_128bits_withSecret
-# undef XXH3_128bits_reset
-# undef XXH3_128bits_reset_withSeed
-# undef XXH3_128bits_reset_withSecret
-# undef XXH3_128bits_reset_withSecretandSeed
-# undef XXH3_128bits_update
-# undef XXH3_128bits_digest
-# undef XXH128_isEqual
-# undef XXH128_cmp
-# undef XXH128_canonicalFromHash
-# undef XXH128_hashFromCanonical
- /* Finally, free the namespace itself */
-# undef XXH_NAMESPACE
-
- /* employ the namespace for XXH_INLINE_ALL */
-# define XXH_NAMESPACE XXH_INLINE_
- /*
- * Some identifiers (enums, type names) are not symbols,
- * but they must nonetheless be renamed to avoid redeclaration.
- * Alternative solution: do not redeclare them.
- * However, this requires some #ifdefs, and has a more dispersed impact.
- * Meanwhile, renaming can be achieved in a single place.
- */
-# define XXH_IPREF(Id) XXH_NAMESPACE ## Id
-# define XXH_OK XXH_IPREF(XXH_OK)
-# define XXH_ERROR XXH_IPREF(XXH_ERROR)
-# define XXH_errorcode XXH_IPREF(XXH_errorcode)
-# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t)
-# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t)
-# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t)
-# define XXH32_state_s XXH_IPREF(XXH32_state_s)
-# define XXH32_state_t XXH_IPREF(XXH32_state_t)
-# define XXH64_state_s XXH_IPREF(XXH64_state_s)
-# define XXH64_state_t XXH_IPREF(XXH64_state_t)
-# define XXH3_state_s XXH_IPREF(XXH3_state_s)
-# define XXH3_state_t XXH_IPREF(XXH3_state_t)
-# define XXH128_hash_t XXH_IPREF(XXH128_hash_t)
- /* Ensure the header is parsed again, even if it was previously included */
-# undef XXHASH_H_5627135585666179
-# undef XXHASH_H_STATIC_13879238742
-#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
-
-
-
-/* ****************************************************************
- * Stable API
- *****************************************************************/
-#ifndef XXHASH_H_5627135585666179
-#define XXHASH_H_5627135585666179 1
-
-
-/*!
- * @defgroup public Public API
- * Contains details on the public xxHash functions.
- * @{
- */
-/* specific declaration modes for Windows */
-#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
-# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
-# ifdef XXH_EXPORT
-# define XXH_PUBLIC_API __declspec(dllexport)
-# elif XXH_IMPORT
-# define XXH_PUBLIC_API __declspec(dllimport)
-# endif
-# else
-# define XXH_PUBLIC_API /* do nothing */
-# endif
-#endif
-
-#ifdef XXH_DOXYGEN
-/*!
- * @brief Emulate a namespace by transparently prefixing all symbols.
- *
- * If you want to include _and expose_ xxHash functions from within your own
- * library, but also want to avoid symbol collisions with other libraries which
- * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix
- * any public symbol from xxhash library with the value of XXH_NAMESPACE
- * (therefore, avoid empty or numeric values).
- *
- * Note that no change is required within the calling program as long as it
- * includes `xxhash.h`: Regular symbol names will be automatically translated
- * by this header.
- */
-# define XXH_NAMESPACE /* YOUR NAME HERE */
-# undef XXH_NAMESPACE
-#endif
-
-#ifdef XXH_NAMESPACE
-# define XXH_CAT(A,B) A##B
-# define XXH_NAME2(A,B) XXH_CAT(A,B)
-# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
-/* XXH32 */
-# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
-# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
-# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
-# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
-# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
-# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
-# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
-# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
-# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
-/* XXH64 */
-# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
-# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
-# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
-# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
-# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
-# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
-# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
-# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
-# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
-/* XXH3_64bits */
-# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
-# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
-# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
-# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed)
-# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
-# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
-# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
-# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
-# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
-# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
-# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed)
-# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
-# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
-# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
-# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed)
-/* XXH3_128bits */
-# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
-# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
-# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
-# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
-# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed)
-# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
-# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
-# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
-# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed)
-# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
-# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
-# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
-# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
-# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
-# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
-#endif
-
-
-/* *************************************
-* Version
-***************************************/
-#define XXH_VERSION_MAJOR 0
-#define XXH_VERSION_MINOR 8
-#define XXH_VERSION_RELEASE 1
-#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
-
-/*!
- * @brief Obtains the xxHash version.
- *
- * This is mostly useful when xxHash is compiled as a shared library,
- * since the returned value comes from the library, as opposed to header file.
- *
- * @return `XXH_VERSION_NUMBER` of the invoked library.
- */
-XXH_PUBLIC_API unsigned XXH_versionNumber (void);
-
-
-/* ****************************
-* Common basic types
-******************************/
-#include <stddef.h> /* size_t */
-typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
-
-
-/*-**********************************************************************
-* 32-bit hash
-************************************************************************/
-#if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */
-/*!
- * @brief An unsigned 32-bit integer.
- *
- * Not necessarily defined to `uint32_t` but functionally equivalent.
- */
-typedef uint32_t XXH32_hash_t;
-
-#elif !defined (__VMS) \
- && (defined (__cplusplus) \
- || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include <stdint.h>
- typedef uint32_t XXH32_hash_t;
-
-#else
-# include <limits.h>
-# if UINT_MAX == 0xFFFFFFFFUL
- typedef unsigned int XXH32_hash_t;
-# else
-# if ULONG_MAX == 0xFFFFFFFFUL
- typedef unsigned long XXH32_hash_t;
-# else
-# error "unsupported platform: need a 32-bit type"
-# endif
-# endif
-#endif
-
-/*!
- * @}
- *
- * @defgroup xxh32_family XXH32 family
- * @ingroup public
- * Contains functions used in the classic 32-bit xxHash algorithm.
- *
- * @note
- * XXH32 is useful for older platforms, with no or poor 64-bit performance.
- * Note that @ref xxh3_family provides competitive speed
- * for both 32-bit and 64-bit systems, and offers true 64/128 bit hash results.
- *
- * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families
- * @see @ref xxh32_impl for implementation details
- * @{
- */
-
-/*!
- * @brief Calculates the 32-bit hash of @p input using xxHash32.
- *
- * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s
- *
- * @param input The block of data to be hashed, at least @p length bytes in size.
- * @param length The length of @p input, in bytes.
- * @param seed The 32-bit seed to alter the hash's output predictably.
- *
- * @pre
- * The memory between @p input and @p input + @p length must be valid,
- * readable, contiguous memory. However, if @p length is `0`, @p input may be
- * `NULL`. In C++, this also must be *TriviallyCopyable*.
- *
- * @return The calculated 32-bit hash value.
- *
- * @see
- * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
- * Direct equivalents for the other variants of xxHash.
- * @see
- * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version.
- */
-XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);
-
-/*!
- * Streaming functions generate the xxHash value from an incremental input.
- * This method is slower than single-call functions, due to state management.
- * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
- *
- * An XXH state must first be allocated using `XXH*_createState()`.
- *
- * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
- *
- * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
- *
- * The function returns an error code, with 0 meaning OK, and any other value
- * meaning there is an error.
- *
- * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
- * This function returns the nn-bits hash as an int or long long.
- *
- * It's still possible to continue inserting input into the hash state after a
- * digest, and generate new hash values later on by invoking `XXH*_digest()`.
- *
- * When done, release the state using `XXH*_freeState()`.
- *
- * Example code for incrementally hashing a file:
- * @code{.c}
- * #include <stdio.h>
- * #include <xxhash.h>
- * #define BUFFER_SIZE 256
- *
- * // Note: XXH64 and XXH3 use the same interface.
- * XXH32_hash_t
- * hashFile(FILE* stream)
- * {
- * XXH32_state_t* state;
- * unsigned char buf[BUFFER_SIZE];
- * size_t amt;
- * XXH32_hash_t hash;
- *
- * state = XXH32_createState(); // Create a state
- * assert(state != NULL); // Error check here
- * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed
- * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) {
- * XXH32_update(state, buf, amt); // Hash the file in chunks
- * }
- * hash = XXH32_digest(state); // Finalize the hash
- * XXH32_freeState(state); // Clean up
- * return hash;
- * }
- * @endcode
- */
-
-/*!
- * @typedef struct XXH32_state_s XXH32_state_t
- * @brief The opaque state struct for the XXH32 streaming API.
- *
- * @see XXH32_state_s for details.
- */
-typedef struct XXH32_state_s XXH32_state_t;
-
-/*!
- * @brief Allocates an @ref XXH32_state_t.
- *
- * Must be freed with XXH32_freeState().
- * @return An allocated XXH32_state_t on success, `NULL` on failure.
- */
-XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
-/*!
- * @brief Frees an @ref XXH32_state_t.
- *
- * Must be allocated with XXH32_createState().
- * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState().
- * @return XXH_OK.
- */
-XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
-/*!
- * @brief Copies one @ref XXH32_state_t to another.
- *
- * @param dst_state The state to copy to.
- * @param src_state The state to copy from.
- * @pre
- * @p dst_state and @p src_state must not be `NULL` and must not overlap.
- */
-XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
-
-/*!
- * @brief Resets an @ref XXH32_state_t to begin a new hash.
- *
- * This function resets and seeds a state. Call it before @ref XXH32_update().
- *
- * @param statePtr The state struct to reset.
- * @param seed The 32-bit seed to alter the hash result predictably.
- *
- * @pre
- * @p statePtr must not be `NULL`.
- *
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
- */
-XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed);
-
-/*!
- * @brief Consumes a block of @p input to an @ref XXH32_state_t.
- *
- * Call this to incrementally consume blocks of data.
- *
- * @param statePtr The state struct to update.
- * @param input The block of data to be hashed, at least @p length bytes in size.
- * @param length The length of @p input, in bytes.
- *
- * @pre
- * @p statePtr must not be `NULL`.
- * @pre
- * The memory between @p input and @p input + @p length must be valid,
- * readable, contiguous memory. However, if @p length is `0`, @p input may be
- * `NULL`. In C++, this also must be *TriviallyCopyable*.
- *
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
- */
-XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
-
-/*!
- * @brief Returns the calculated hash value from an @ref XXH32_state_t.
- *
- * @note
- * Calling XXH32_digest() will not affect @p statePtr, so you can update,
- * digest, and update again.
- *
- * @param statePtr The state struct to calculate the hash from.
- *
- * @pre
- * @p statePtr must not be `NULL`.
- *
- * @return The calculated xxHash32 value from that state.
- */
-XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
-
-/******* Canonical representation *******/
-
-/*
- * The default return values from XXH functions are unsigned 32 and 64 bit
- * integers.
- * This the simplest and fastest format for further post-processing.
- *
- * However, this leaves open the question of what is the order on the byte level,
- * since little and big endian conventions will store the same number differently.
- *
- * The canonical representation settles this issue by mandating big-endian
- * convention, the same convention as human-readable numbers (large digits first).
- *
- * When writing hash values to storage, sending them over a network, or printing
- * them, it's highly recommended to use the canonical representation to ensure
- * portability across a wider range of systems, present and future.
- *
- * The following functions allow transformation of hash values to and from
- * canonical format.
- */
-
-/*!
- * @brief Canonical (big endian) representation of @ref XXH32_hash_t.
- */
-typedef struct {
- unsigned char digest[4]; /*!< Hash bytes, big endian */
-} XXH32_canonical_t;
-
-/*!
- * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t.
- *
- * @param dst The @ref XXH32_canonical_t pointer to be stored to.
- * @param hash The @ref XXH32_hash_t to be converted.
- *
- * @pre
- * @p dst must not be `NULL`.
- */
-XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
-
-/*!
- * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t.
- *
- * @param src The @ref XXH32_canonical_t to convert.
- *
- * @pre
- * @p src must not be `NULL`.
- *
- * @return The converted hash.
- */
-XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
-
-
-#ifdef __has_attribute
-# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x)
-#else
-# define XXH_HAS_ATTRIBUTE(x) 0
-#endif
-
-/* C-language Attributes are added in C23. */
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute)
-# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
-#else
-# define XXH_HAS_C_ATTRIBUTE(x) 0
-#endif
-
-#if defined(__cplusplus) && defined(__has_cpp_attribute)
-# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
-#else
-# define XXH_HAS_CPP_ATTRIBUTE(x) 0
-#endif
-
-/*
-Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute
-introduced in CPP17 and C23.
-CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough
-C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough
-*/
-#if XXH_HAS_C_ATTRIBUTE(x)
-# define XXH_FALLTHROUGH [[fallthrough]]
-#elif XXH_HAS_CPP_ATTRIBUTE(x)
-# define XXH_FALLTHROUGH [[fallthrough]]
-#elif XXH_HAS_ATTRIBUTE(__fallthrough__)
-# define XXH_FALLTHROUGH __attribute__ ((fallthrough))
-#else
-# define XXH_FALLTHROUGH
-#endif
-
-/*!
- * @}
- * @ingroup public
- * @{
- */
-
-#ifndef XXH_NO_LONG_LONG
-/*-**********************************************************************
-* 64-bit hash
-************************************************************************/
-#if defined(XXH_DOXYGEN) /* don't include <stdint.h> */
-/*!
- * @brief An unsigned 64-bit integer.
- *
- * Not necessarily defined to `uint64_t` but functionally equivalent.
- */
-typedef uint64_t XXH64_hash_t;
-#elif !defined (__VMS) \
- && (defined (__cplusplus) \
- || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include <stdint.h>
- typedef uint64_t XXH64_hash_t;
-#else
-# include <limits.h>
-# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL
- /* LP64 ABI says uint64_t is unsigned long */
- typedef unsigned long XXH64_hash_t;
-# else
- /* the following type must have a width of 64-bit */
- typedef unsigned long long XXH64_hash_t;
-# endif
-#endif
-
-/*!
- * @}
- *
- * @defgroup xxh64_family XXH64 family
- * @ingroup public
- * @{
- * Contains functions used in the classic 64-bit xxHash algorithm.
- *
- * @note
- * XXH3 provides competitive speed for both 32-bit and 64-bit systems,
- * and offers true 64/128 bit hash results.
- * It provides better speed for systems with vector processing capabilities.
- */
-
-
-/*!
- * @brief Calculates the 64-bit hash of @p input using xxHash64.
- *
- * This function usually runs faster on 64-bit systems, but slower on 32-bit
- * systems (see benchmark).
- *
- * @param input The block of data to be hashed, at least @p length bytes in size.
- * @param length The length of @p input, in bytes.
- * @param seed The 64-bit seed to alter the hash's output predictably.
- *
- * @pre
- * The memory between @p input and @p input + @p length must be valid,
- * readable, contiguous memory. However, if @p length is `0`, @p input may be
- * `NULL`. In C++, this also must be *TriviallyCopyable*.
- *
- * @return The calculated 64-bit hash.
- *
- * @see
- * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
- * Direct equivalents for the other variants of xxHash.
- * @see
- * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version.
- */
-XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed);
-
-/******* Streaming *******/
-/*!
- * @brief The opaque state struct for the XXH64 streaming API.
- *
- * @see XXH64_state_s for details.
- */
-typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
-XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
-XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
-XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
-
-XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed);
-XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
-XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
-
-/******* Canonical representation *******/
-typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t;
-XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
-XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
-
-#ifndef XXH_NO_XXH3
-/*!
- * @}
- * ************************************************************************
- * @defgroup xxh3_family XXH3 family
- * @ingroup public
- * @{
- *
- * XXH3 is a more recent hash algorithm featuring:
- * - Improved speed for both small and large inputs
- * - True 64-bit and 128-bit outputs
- * - SIMD acceleration
- * - Improved 32-bit viability
- *
- * Speed analysis methodology is explained here:
- *
- * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
- *
- * Compared to XXH64, expect XXH3 to run approximately
- * ~2x faster on large inputs and >3x faster on small ones,
- * exact differences vary depending on platform.
- *
- * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic,
- * but does not require it.
- * Any 32-bit and 64-bit targets that can run XXH32 smoothly
- * can run XXH3 at competitive speeds, even without vector support.
- * Further details are explained in the implementation.
- *
- * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8,
- * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro.
- *
- * XXH3 implementation is portable:
- * it has a generic C90 formulation that can be compiled on any platform,
- * all implementations generage exactly the same hash value on all platforms.
- * Starting from v0.8.0, it's also labelled "stable", meaning that
- * any future version will also generate the same hash value.
- *
- * XXH3 offers 2 variants, _64bits and _128bits.
- *
- * When only 64 bits are needed, prefer invoking the _64bits variant, as it
- * reduces the amount of mixing, resulting in faster speed on small inputs.
- * It's also generally simpler to manipulate a scalar return type than a struct.
- *
- * The API supports one-shot hashing, streaming mode, and custom secrets.
- */
-
-/*-**********************************************************************
-* XXH3 64-bit variant
-************************************************************************/
-
-/* XXH3_64bits():
- * default 64-bit variant, using default secret and default seed of 0.
- * It's the fastest variant. */
-XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len);
-
-/*
- * XXH3_64bits_withSeed():
- * This variant generates a custom secret on the fly
- * based on default secret altered using the `seed` value.
- * While this operation is decently fast, note that it's not completely free.
- * Note: seed==0 produces the same results as XXH3_64bits().
- */
-XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);
-
-/*!
- * The bare minimum size for a custom secret.
- *
- * @see
- * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(),
- * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret().
- */
-#define XXH3_SECRET_SIZE_MIN 136
-
-/*
- * XXH3_64bits_withSecret():
- * It's possible to provide any blob of bytes as a "secret" to generate the hash.
- * This makes it more difficult for an external actor to prepare an intentional collision.
- * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN).
- * However, the quality of the secret impacts the dispersion of the hash algorithm.
- * Therefore, the secret _must_ look like a bunch of random bytes.
- * Avoid "trivial" or structured data such as repeated sequences or a text document.
- * Whenever in doubt about the "randomness" of the blob of bytes,
- * consider employing "XXH3_generateSecret()" instead (see below).
- * It will generate a proper high entropy secret derived from the blob of bytes.
- * Another advantage of using XXH3_generateSecret() is that
- * it guarantees that all bits within the initial blob of bytes
- * will impact every bit of the output.
- * This is not necessarily the case when using the blob of bytes directly
- * because, when hashing _small_ inputs, only a portion of the secret is employed.
- */
-XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize);
-
-
-/******* Streaming *******/
-/*
- * Streaming requires state maintenance.
- * This operation costs memory and CPU.
- * As a consequence, streaming is slower than one-shot hashing.
- * For better performance, prefer one-shot functions whenever applicable.
- */
-
-/*!
- * @brief The state struct for the XXH3 streaming API.
- *
- * @see XXH3_state_s for details.
- */
-typedef struct XXH3_state_s XXH3_state_t;
-XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void);
-XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr);
-XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state);
-
-/*
- * XXH3_64bits_reset():
- * Initialize with default parameters.
- * digest will be equivalent to `XXH3_64bits()`.
- */
-XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr);
-/*
- * XXH3_64bits_reset_withSeed():
- * Generate a custom secret from `seed`, and store it into `statePtr`.
- * digest will be equivalent to `XXH3_64bits_withSeed()`.
- */
-XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed);
-/*
- * XXH3_64bits_reset_withSecret():
- * `secret` is referenced, it _must outlive_ the hash streaming session.
- * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`,
- * and the quality of produced hash values depends on secret's entropy
- * (secret's content should look like a bunch of random bytes).
- * When in doubt about the randomness of a candidate `secret`,
- * consider employing `XXH3_generateSecret()` instead (see below).
- */
-XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize);
-
-XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length);
-XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr);
-
-/* note : canonical representation of XXH3 is the same as XXH64
- * since they both produce XXH64_hash_t values */
-
-
-/*-**********************************************************************
-* XXH3 128-bit variant
-************************************************************************/
-
-/*!
- * @brief The return value from 128-bit hashes.
- *
- * Stored in little endian order, although the fields themselves are in native
- * endianness.
- */
-typedef struct {
- XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */
- XXH64_hash_t high64; /*!< `value >> 64` */
-} XXH128_hash_t;
-
-XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len);
-XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);
-XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize);
-
-/******* Streaming *******/
-/*
- * Streaming requires state maintenance.
- * This operation costs memory and CPU.
- * As a consequence, streaming is slower than one-shot hashing.
- * For better performance, prefer one-shot functions whenever applicable.
- *
- * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits().
- * Use already declared XXH3_createState() and XXH3_freeState().
- *
- * All reset and streaming functions have same meaning as their 64-bit counterpart.
- */
-
-XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr);
-XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed);
-XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize);
-
-XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length);
-XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr);
-
-/* Following helper functions make it possible to compare XXH128_hast_t values.
- * Since XXH128_hash_t is a structure, this capability is not offered by the language.
- * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */
-
-/*!
- * XXH128_isEqual():
- * Return: 1 if `h1` and `h2` are equal, 0 if they are not.
- */
-XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);
-
-/*!
- * XXH128_cmp():
- *
- * This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
- *
- * return: >0 if *h128_1 > *h128_2
- * =0 if *h128_1 == *h128_2
- * <0 if *h128_1 < *h128_2
- */
-XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2);
-
-
-/******* Canonical representation *******/
-typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t;
-XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash);
-XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src);
-
-
-#endif /* !XXH_NO_XXH3 */
-#endif /* XXH_NO_LONG_LONG */
-
-/*!
- * @}
- */
-#endif /* XXHASH_H_5627135585666179 */
-
-
-
-#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742)
-#define XXHASH_H_STATIC_13879238742
-/* ****************************************************************************
- * This section contains declarations which are not guaranteed to remain stable.
- * They may change in future versions, becoming incompatible with a different
- * version of the library.
- * These declarations should only be used with static linking.
- * Never use them in association with dynamic linking!
- ***************************************************************************** */
-
-/*
- * These definitions are only present to allow static allocation
- * of XXH states, on stack or in a struct, for example.
- * Never **ever** access their members directly.
- */
-
-/*!
- * @internal
- * @brief Structure for XXH32 streaming API.
- *
- * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
- * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
- * an opaque type. This allows fields to safely be changed.
- *
- * Typedef'd to @ref XXH32_state_t.
- * Do not access the members of this struct directly.
- * @see XXH64_state_s, XXH3_state_s
- */
-struct XXH32_state_s {
- XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */
- XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */
- XXH32_hash_t v[4]; /*!< Accumulator lanes */
- XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */
- XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */
- XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */
-}; /* typedef'd to XXH32_state_t */
-
-
-#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */
-
-/*!
- * @internal
- * @brief Structure for XXH64 streaming API.
- *
- * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
- * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
- * an opaque type. This allows fields to safely be changed.
- *
- * Typedef'd to @ref XXH64_state_t.
- * Do not access the members of this struct directly.
- * @see XXH32_state_s, XXH3_state_s
- */
-struct XXH64_state_s {
- XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */
- XXH64_hash_t v[4]; /*!< Accumulator lanes */
- XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */
- XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */
- XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/
- XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */
-}; /* typedef'd to XXH64_state_t */
-
-
-#ifndef XXH_NO_XXH3
-
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */
-# include <stdalign.h>
-# define XXH_ALIGN(n) alignas(n)
-#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
-/* In C++ alignas() is a keyword */
-# define XXH_ALIGN(n) alignas(n)
-#elif defined(__GNUC__)
-# define XXH_ALIGN(n) __attribute__ ((aligned(n)))
-#elif defined(_MSC_VER)
-# define XXH_ALIGN(n) __declspec(align(n))
-#else
-# define XXH_ALIGN(n) /* disabled */
-#endif
-
-/* Old GCC versions only accept the attribute after the type in structures. */
-#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \
- && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \
- && defined(__GNUC__)
-# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align)
-#else
-# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type
-#endif
-
-/*!
- * @brief The size of the internal XXH3 buffer.
- *
- * This is the optimal update size for incremental hashing.
- *
- * @see XXH3_64b_update(), XXH3_128b_update().
- */
-#define XXH3_INTERNALBUFFER_SIZE 256
-
-/*!
- * @brief Default size of the secret buffer (and @ref XXH3_kSecret).
- *
- * This is the size used in @ref XXH3_kSecret and the seeded functions.
- *
- * Not to be confused with @ref XXH3_SECRET_SIZE_MIN.
- */
-#define XXH3_SECRET_DEFAULT_SIZE 192
-
-/*!
- * @internal
- * @brief Structure for XXH3 streaming API.
- *
- * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
- * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined.
- * Otherwise it is an opaque type.
- * Never use this definition in combination with dynamic library.
- * This allows fields to safely be changed in the future.
- *
- * @note ** This structure has a strict alignment requirement of 64 bytes!! **
- * Do not allocate this with `malloc()` or `new`,
- * it will not be sufficiently aligned.
- * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation.
- *
- * Typedef'd to @ref XXH3_state_t.
- * Do never access the members of this struct directly.
- *
- * @see XXH3_INITSTATE() for stack initialization.
- * @see XXH3_createState(), XXH3_freeState().
- * @see XXH32_state_s, XXH64_state_s
- */
-struct XXH3_state_s {
- XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]);
- /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */
- XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]);
- /*!< Used to store a custom secret generated from a seed. */
- XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]);
- /*!< The internal buffer. @see XXH32_state_s::mem32 */
- XXH32_hash_t bufferedSize;
- /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */
- XXH32_hash_t useSeed;
- /*!< Reserved field. Needed for padding on 64-bit. */
- size_t nbStripesSoFar;
- /*!< Number or stripes processed. */
- XXH64_hash_t totalLen;
- /*!< Total length hashed. 64-bit even on 32-bit targets. */
- size_t nbStripesPerBlock;
- /*!< Number of stripes per block. */
- size_t secretLimit;
- /*!< Size of @ref customSecret or @ref extSecret */
- XXH64_hash_t seed;
- /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */
- XXH64_hash_t reserved64;
- /*!< Reserved field. */
- const unsigned char* extSecret;
- /*!< Reference to an external secret for the _withSecret variants, NULL
- * for other variants. */
- /* note: there may be some padding at the end due to alignment on 64 bytes */
-}; /* typedef'd to XXH3_state_t */
-
-#undef XXH_ALIGN_MEMBER
-
-/*!
- * @brief Initializes a stack-allocated `XXH3_state_s`.
- *
- * When the @ref XXH3_state_t structure is merely emplaced on stack,
- * it should be initialized with XXH3_INITSTATE() or a memset()
- * in case its first reset uses XXH3_NNbits_reset_withSeed().
- * This init can be omitted if the first reset uses default or _withSecret mode.
- * This operation isn't necessary when the state is created with XXH3_createState().
- * Note that this doesn't prepare the state for a streaming operation,
- * it's still necessary to use XXH3_NNbits_reset*() afterwards.
- */
-#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; }
-
-
-/* XXH128() :
- * simple alias to pre-selected XXH3_128bits variant
- */
-XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed);
-
-
-/* === Experimental API === */
-/* Symbols defined below must be considered tied to a specific library version. */
-
-/*
- * XXH3_generateSecret():
- *
- * Derive a high-entropy secret from any user-defined content, named customSeed.
- * The generated secret can be used in combination with `*_withSecret()` functions.
- * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed,
- * as it becomes much more difficult for an external actor to guess how to impact the calculation logic.
- *
- * The function accepts as input a custom seed of any length and any content,
- * and derives from it a high-entropy secret of length @secretSize
- * into an already allocated buffer @secretBuffer.
- * @secretSize must be >= XXH3_SECRET_SIZE_MIN
- *
- * The generated secret can then be used with any `*_withSecret()` variant.
- * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`,
- * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()`
- * are part of this list. They all accept a `secret` parameter
- * which must be large enough for implementation reasons (>= XXH3_SECRET_SIZE_MIN)
- * _and_ feature very high entropy (consist of random-looking bytes).
- * These conditions can be a high bar to meet, so
- * XXH3_generateSecret() can be employed to ensure proper quality.
- *
- * customSeed can be anything. It can have any size, even small ones,
- * and its content can be anything, even "poor entropy" sources such as a bunch of zeroes.
- * The resulting `secret` will nonetheless provide all required qualities.
- *
- * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior.
- */
-XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize);
-
-
-/*
- * XXH3_generateSecret_fromSeed():
- *
- * Generate the same secret as the _withSeed() variants.
- *
- * The resulting secret has a length of XXH3_SECRET_DEFAULT_SIZE (necessarily).
- * @secretBuffer must be already allocated, of size at least XXH3_SECRET_DEFAULT_SIZE bytes.
- *
- * The generated secret can be used in combination with
- *`*_withSecret()` and `_withSecretandSeed()` variants.
- * This generator is notably useful in combination with `_withSecretandSeed()`,
- * as a way to emulate a faster `_withSeed()` variant.
- */
-XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed);
-
-/*
- * *_withSecretandSeed() :
- * These variants generate hash values using either
- * @seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes)
- * or @secret for "large" keys (>= XXH3_MIDSIZE_MAX).
- *
- * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`.
- * `_withSeed()` has to generate the secret on the fly for "large" keys.
- * It's fast, but can be perceptible for "not so large" keys (< 1 KB).
- * `_withSecret()` has to generate the masks on the fly for "small" keys,
- * which requires more instructions than _withSeed() variants.
- * Therefore, _withSecretandSeed variant combines the best of both worlds.
- *
- * When @secret has been generated by XXH3_generateSecret_fromSeed(),
- * this variant produces *exactly* the same results as `_withSeed()` variant,
- * hence offering only a pure speed benefit on "large" input,
- * by skipping the need to regenerate the secret for every large input.
- *
- * Another usage scenario is to hash the secret to a 64-bit hash value,
- * for example with XXH3_64bits(), which then becomes the seed,
- * and then employ both the seed and the secret in _withSecretandSeed().
- * On top of speed, an added benefit is that each bit in the secret
- * has a 50% chance to swap each bit in the output,
- * via its impact to the seed.
- * This is not guaranteed when using the secret directly in "small data" scenarios,
- * because only portions of the secret are employed for small data.
- */
-XXH_PUBLIC_API XXH64_hash_t
-XXH3_64bits_withSecretandSeed(const void* data, size_t len,
- const void* secret, size_t secretSize,
- XXH64_hash_t seed);
-
-XXH_PUBLIC_API XXH128_hash_t
-XXH3_128bits_withSecretandSeed(const void* data, size_t len,
- const void* secret, size_t secretSize,
- XXH64_hash_t seed64);
-
-XXH_PUBLIC_API XXH_errorcode
-XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr,
- const void* secret, size_t secretSize,
- XXH64_hash_t seed64);
-
-XXH_PUBLIC_API XXH_errorcode
-XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr,
- const void* secret, size_t secretSize,
- XXH64_hash_t seed64);
-
-
-#endif /* XXH_NO_XXH3 */
-#endif /* XXH_NO_LONG_LONG */
-#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
-# define XXH_IMPLEMENTATION
-#endif
-
-#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */
-
-
-/* ======================================================================== */
-/* ======================================================================== */
-/* ======================================================================== */
-
-
-/*-**********************************************************************
- * xxHash implementation
- *-**********************************************************************
- * xxHash's implementation used to be hosted inside xxhash.c.
- *
- * However, inlining requires implementation to be visible to the compiler,
- * hence be included alongside the header.
- * Previously, implementation was hosted inside xxhash.c,
- * which was then #included when inlining was activated.
- * This construction created issues with a few build and install systems,
- * as it required xxhash.c to be stored in /include directory.
- *
- * xxHash implementation is now directly integrated within xxhash.h.
- * As a consequence, xxhash.c is no longer needed in /include.
- *
- * xxhash.c is still available and is still useful.
- * In a "normal" setup, when xxhash is not inlined,
- * xxhash.h only exposes the prototypes and public symbols,
- * while xxhash.c can be built into an object file xxhash.o
- * which can then be linked into the final binary.
- ************************************************************************/
-
-#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \
- || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387)
-# define XXH_IMPLEM_13a8737387
-
-/* *************************************
-* Tuning parameters
-***************************************/
-
-/*!
- * @defgroup tuning Tuning parameters
- * @{
- *
- * Various macros to control xxHash's behavior.
- */
-#ifdef XXH_DOXYGEN
-/*!
- * @brief Define this to disable 64-bit code.
- *
- * Useful if only using the @ref xxh32_family and you have a strict C90 compiler.
- */
-# define XXH_NO_LONG_LONG
-# undef XXH_NO_LONG_LONG /* don't actually */
-/*!
- * @brief Controls how unaligned memory is accessed.
- *
- * By default, access to unaligned memory is controlled by `memcpy()`, which is
- * safe and portable.
- *
- * Unfortunately, on some target/compiler combinations, the generated assembly
- * is sub-optimal.
- *
- * The below switch allow selection of a different access method
- * in the search for improved performance.
- *
- * @par Possible options:
- *
- * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy`
- * @par
- * Use `memcpy()`. Safe and portable. Note that most modern compilers will
- * eliminate the function call and treat it as an unaligned access.
- *
- * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))`
- * @par
- * Depends on compiler extensions and is therefore not portable.
- * This method is safe _if_ your compiler supports it,
- * and *generally* as fast or faster than `memcpy`.
- *
- * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast
- * @par
- * Casts directly and dereferences. This method doesn't depend on the
- * compiler, but it violates the C standard as it directly dereferences an
- * unaligned pointer. It can generate buggy code on targets which do not
- * support unaligned memory accesses, but in some circumstances, it's the
- * only known way to get the most performance.
- *
- * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift
- * @par
- * Also portable. This can generate the best code on old compilers which don't
- * inline small `memcpy()` calls, and it might also be faster on big-endian
- * systems which lack a native byteswap instruction. However, some compilers
- * will emit literal byteshifts even if the target supports unaligned access.
- * .
- *
- * @warning
- * Methods 1 and 2 rely on implementation-defined behavior. Use these with
- * care, as what works on one compiler/platform/optimization level may cause
- * another to read garbage data or even crash.
- *
- * See http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
- *
- * Prefer these methods in priority order (0 > 3 > 1 > 2)
- */
-# define XXH_FORCE_MEMORY_ACCESS 0
-
-/*!
- * @def XXH_FORCE_ALIGN_CHECK
- * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32()
- * and XXH64() only).
- *
- * This is an important performance trick for architectures without decent
- * unaligned memory access performance.
- *
- * It checks for input alignment, and when conditions are met, uses a "fast
- * path" employing direct 32-bit/64-bit reads, resulting in _dramatically
- * faster_ read speed.
- *
- * The check costs one initial branch per hash, which is generally negligible,
- * but not zero.
- *
- * Moreover, it's not useful to generate an additional code path if memory
- * access uses the same instruction for both aligned and unaligned
- * addresses (e.g. x86 and aarch64).
- *
- * In these cases, the alignment check can be removed by setting this macro to 0.
- * Then the code will always use unaligned memory access.
- * Align check is automatically disabled on x86, x64 & arm64,
- * which are platforms known to offer good unaligned memory accesses performance.
- *
- * This option does not affect XXH3 (only XXH32 and XXH64).
- */
-# define XXH_FORCE_ALIGN_CHECK 0
-
-/*!
- * @def XXH_NO_INLINE_HINTS
- * @brief When non-zero, sets all functions to `static`.
- *
- * By default, xxHash tries to force the compiler to inline almost all internal
- * functions.
- *
- * This can usually improve performance due to reduced jumping and improved
- * constant folding, but significantly increases the size of the binary which
- * might not be favorable.
- *
- * Additionally, sometimes the forced inlining can be detrimental to performance,
- * depending on the architecture.
- *
- * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the
- * compiler full control on whether to inline or not.
- *
- * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using
- * -fno-inline with GCC or Clang, this will automatically be defined.
- */
-# define XXH_NO_INLINE_HINTS 0
-
-/*!
- * @def XXH32_ENDJMP
- * @brief Whether to use a jump for `XXH32_finalize`.
- *
- * For performance, `XXH32_finalize` uses multiple branches in the finalizer.
- * This is generally preferable for performance,
- * but depending on exact architecture, a jmp may be preferable.
- *
- * This setting is only possibly making a difference for very small inputs.
- */
-# define XXH32_ENDJMP 0
-
-/*!
- * @internal
- * @brief Redefines old internal names.
- *
- * For compatibility with code that uses xxHash's internals before the names
- * were changed to improve namespacing. There is no other reason to use this.
- */
-# define XXH_OLD_NAMES
-# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */
-#endif /* XXH_DOXYGEN */
-/*!
- * @}
- */
-
-#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
- /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */
-# if !defined(__clang__) && \
-( \
- (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \
- ( \
- defined(__GNUC__) && ( \
- (defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \
- ( \
- defined(__mips__) && \
- (__mips <= 5 || __mips_isa_rev < 6) && \
- (!defined(__mips16) || defined(__mips_mips16e2)) \
- ) \
- ) \
- ) \
-)
-# define XXH_FORCE_MEMORY_ACCESS 1
-# endif
-#endif
-
-#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
-# if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) \
- || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */
-# define XXH_FORCE_ALIGN_CHECK 0
-# else
-# define XXH_FORCE_ALIGN_CHECK 1
-# endif
-#endif
-
-#ifndef XXH_NO_INLINE_HINTS
-# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \
- || defined(__NO_INLINE__) /* -O0, -fno-inline */
-# define XXH_NO_INLINE_HINTS 1
-# else
-# define XXH_NO_INLINE_HINTS 0
-# endif
-#endif
-
-#ifndef XXH32_ENDJMP
-/* generally preferable for performance */
-# define XXH32_ENDJMP 0
-#endif
-
-/*!
- * @defgroup impl Implementation
- * @{
- */
-
-
-/* *************************************
-* Includes & Memory related functions
-***************************************/
-/* Modify the local functions below should you wish to use some other memory routines */
-/* for ZSTD_malloc(), ZSTD_free() */
-#define ZSTD_DEPS_NEED_MALLOC
-#include "zstd_deps.h" /* size_t, ZSTD_malloc, ZSTD_free, ZSTD_memcpy */
-static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); }
-static void XXH_free (void* p) { ZSTD_free(p); }
-static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); }
-
-
-/* *************************************
-* Compiler Specific Options
-***************************************/
-#ifdef _MSC_VER /* Visual Studio warning fix */
-# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#endif
-
-#if XXH_NO_INLINE_HINTS /* disable inlining hints */
-# if defined(__GNUC__) || defined(__clang__)
-# define XXH_FORCE_INLINE static __attribute__((unused))
-# else
-# define XXH_FORCE_INLINE static
-# endif
-# define XXH_NO_INLINE static
-/* enable inlining hints */
-#elif defined(__GNUC__) || defined(__clang__)
-# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused))
-# define XXH_NO_INLINE static __attribute__((noinline))
-#elif defined(_MSC_VER) /* Visual Studio */
-# define XXH_FORCE_INLINE static __forceinline
-# define XXH_NO_INLINE static __declspec(noinline)
-#elif defined (__cplusplus) \
- || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */
-# define XXH_FORCE_INLINE static inline
-# define XXH_NO_INLINE static
-#else
-# define XXH_FORCE_INLINE static
-# define XXH_NO_INLINE static
-#endif
-
-
-
-/* *************************************
-* Debug
-***************************************/
-/*!
- * @ingroup tuning
- * @def XXH_DEBUGLEVEL
- * @brief Sets the debugging level.
- *
- * XXH_DEBUGLEVEL is expected to be defined externally, typically via the
- * compiler's command line options. The value must be a number.
- */
-#ifndef XXH_DEBUGLEVEL
-# ifdef DEBUGLEVEL /* backwards compat */
-# define XXH_DEBUGLEVEL DEBUGLEVEL
-# else
-# define XXH_DEBUGLEVEL 0
-# endif
-#endif
-
-#if (XXH_DEBUGLEVEL>=1)
-# include <assert.h> /* note: can still be disabled with NDEBUG */
-# define XXH_ASSERT(c) assert(c)
-#else
-# define XXH_ASSERT(c) ((void)0)
-#endif
-
-/* note: use after variable declarations */
-#ifndef XXH_STATIC_ASSERT
-# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
-# include <assert.h>
-# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
-# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */
-# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
-# else
-# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0)
-# endif
-# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c)
-#endif
-
-/*!
- * @internal
- * @def XXH_COMPILER_GUARD(var)
- * @brief Used to prevent unwanted optimizations for @p var.
- *
- * It uses an empty GCC inline assembly statement with a register constraint
- * which forces @p var into a general purpose register (eg eax, ebx, ecx
- * on x86) and marks it as modified.
- *
- * This is used in a few places to avoid unwanted autovectorization (e.g.
- * XXH32_round()). All vectorization we want is explicit via intrinsics,
- * and _usually_ isn't wanted elsewhere.
- *
- * We also use it to prevent unwanted constant folding for AArch64 in
- * XXH3_initCustomSecret_scalar().
- */
-#if defined(__GNUC__) || defined(__clang__)
-# define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r" (var))
-#else
-# define XXH_COMPILER_GUARD(var) ((void)0)
-#endif
-
-/* *************************************
-* Basic Types
-***************************************/
-#if !defined (__VMS) \
- && (defined (__cplusplus) \
- || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include <stdint.h>
- typedef uint8_t xxh_u8;
-#else
- typedef unsigned char xxh_u8;
-#endif
-typedef XXH32_hash_t xxh_u32;
-
-#ifdef XXH_OLD_NAMES
-# define BYTE xxh_u8
-# define U8 xxh_u8
-# define U32 xxh_u32
-#endif
-
-/* *** Memory access *** */
-
-/*!
- * @internal
- * @fn xxh_u32 XXH_read32(const void* ptr)
- * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness.
- *
- * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
- *
- * @param ptr The pointer to read from.
- * @return The 32-bit native endian integer from the bytes at @p ptr.
- */
-
-/*!
- * @internal
- * @fn xxh_u32 XXH_readLE32(const void* ptr)
- * @brief Reads an unaligned 32-bit little endian integer from @p ptr.
- *
- * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
- *
- * @param ptr The pointer to read from.
- * @return The 32-bit little endian integer from the bytes at @p ptr.
- */
-
-/*!
- * @internal
- * @fn xxh_u32 XXH_readBE32(const void* ptr)
- * @brief Reads an unaligned 32-bit big endian integer from @p ptr.
- *
- * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
- *
- * @param ptr The pointer to read from.
- * @return The 32-bit big endian integer from the bytes at @p ptr.
- */
-
-/*!
- * @internal
- * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align)
- * @brief Like @ref XXH_readLE32(), but has an option for aligned reads.
- *
- * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
- * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is
- * always @ref XXH_alignment::XXH_unaligned.
- *
- * @param ptr The pointer to read from.
- * @param align Whether @p ptr is aligned.
- * @pre
- * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte
- * aligned.
- * @return The 32-bit little endian integer from the bytes at @p ptr.
- */
-
-#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
-/*
- * Manual byteshift. Best for old compilers which don't inline memcpy.
- * We actually directly use XXH_readLE32 and XXH_readBE32.
- */
-#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
-
-/*
- * Force direct memory access. Only works on CPU which support unaligned memory
- * access in hardware.
- */
-static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; }
-
-#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
-
-/*
- * __pack instructions are safer but compiler specific, hence potentially
- * problematic for some compilers.
- *
- * Currently only defined for GCC and ICC.
- */
-#ifdef XXH_OLD_NAMES
-typedef union { xxh_u32 u32; } __attribute__((packed)) unalign;
-#endif
-static xxh_u32 XXH_read32(const void* ptr)
-{
- typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign;
- return ((const xxh_unalign*)ptr)->u32;
-}
-
-#else
-
-/*
- * Portable and safe solution. Generally efficient.
- * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
- */
-static xxh_u32 XXH_read32(const void* memPtr)
-{
- xxh_u32 val;
- XXH_memcpy(&val, memPtr, sizeof(val));
- return val;
-}
-
-#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
-
-
-/* *** Endianness *** */
-
-/*!
- * @ingroup tuning
- * @def XXH_CPU_LITTLE_ENDIAN
- * @brief Whether the target is little endian.
- *
- * Defined to 1 if the target is little endian, or 0 if it is big endian.
- * It can be defined externally, for example on the compiler command line.
- *
- * If it is not defined,
- * a runtime check (which is usually constant folded) is used instead.
- *
- * @note
- * This is not necessarily defined to an integer constant.
- *
- * @see XXH_isLittleEndian() for the runtime check.
- */
-#ifndef XXH_CPU_LITTLE_ENDIAN
-/*
- * Try to detect endianness automatically, to avoid the nonstandard behavior
- * in `XXH_isLittleEndian()`
- */
-# if defined(_WIN32) /* Windows is always little endian */ \
- || defined(__LITTLE_ENDIAN__) \
- || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
-# define XXH_CPU_LITTLE_ENDIAN 1
-# elif defined(__BIG_ENDIAN__) \
- || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
-# define XXH_CPU_LITTLE_ENDIAN 0
-# else
-/*!
- * @internal
- * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN.
- *
- * Most compilers will constant fold this.
- */
-static int XXH_isLittleEndian(void)
-{
- /*
- * Portable and well-defined behavior.
- * Don't use static: it is detrimental to performance.
- */
- const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 };
- return one.c[0];
-}
-# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian()
-# endif
-#endif
-
-
-
-
-/* ****************************************
-* Compiler-specific Functions and Macros
-******************************************/
-#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-
-#ifdef __has_builtin
-# define XXH_HAS_BUILTIN(x) __has_builtin(x)
-#else
-# define XXH_HAS_BUILTIN(x) 0
-#endif
-
-/*!
- * @internal
- * @def XXH_rotl32(x,r)
- * @brief 32-bit rotate left.
- *
- * @param x The 32-bit integer to be rotated.
- * @param r The number of bits to rotate.
- * @pre
- * @p r > 0 && @p r < 32
- * @note
- * @p x and @p r may be evaluated multiple times.
- * @return The rotated result.
- */
-#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \
- && XXH_HAS_BUILTIN(__builtin_rotateleft64)
-# define XXH_rotl32 __builtin_rotateleft32
-# define XXH_rotl64 __builtin_rotateleft64
-/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
-#elif defined(_MSC_VER)
-# define XXH_rotl32(x,r) _rotl(x,r)
-# define XXH_rotl64(x,r) _rotl64(x,r)
-#else
-# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
-# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r))))
-#endif
-
-/*!
- * @internal
- * @fn xxh_u32 XXH_swap32(xxh_u32 x)
- * @brief A 32-bit byteswap.
- *
- * @param x The 32-bit integer to byteswap.
- * @return @p x, byteswapped.
- */
-#if defined(_MSC_VER) /* Visual Studio */
-# define XXH_swap32 _byteswap_ulong
-#elif XXH_GCC_VERSION >= 403
-# define XXH_swap32 __builtin_bswap32
-#else
-static xxh_u32 XXH_swap32 (xxh_u32 x)
-{
- return ((x << 24) & 0xff000000 ) |
- ((x << 8) & 0x00ff0000 ) |
- ((x >> 8) & 0x0000ff00 ) |
- ((x >> 24) & 0x000000ff );
-}
-#endif
-
-
-/* ***************************
-* Memory reads
-*****************************/
-
-/*!
- * @internal
- * @brief Enum to indicate whether a pointer is aligned.
- */
-typedef enum {
- XXH_aligned, /*!< Aligned */
- XXH_unaligned /*!< Possibly unaligned */
-} XXH_alignment;
-
-/*
- * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load.
- *
- * This is ideal for older compilers which don't inline memcpy.
- */
-#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
-
-XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr)
-{
- const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
- return bytePtr[0]
- | ((xxh_u32)bytePtr[1] << 8)
- | ((xxh_u32)bytePtr[2] << 16)
- | ((xxh_u32)bytePtr[3] << 24);
-}
-
-XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr)
-{
- const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
- return bytePtr[3]
- | ((xxh_u32)bytePtr[2] << 8)
- | ((xxh_u32)bytePtr[1] << 16)
- | ((xxh_u32)bytePtr[0] << 24);
-}
-
-#else
-XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr)
-{
- return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
-}
-
-static xxh_u32 XXH_readBE32(const void* ptr)
-{
- return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
-}
-#endif
-
-XXH_FORCE_INLINE xxh_u32
-XXH_readLE32_align(const void* ptr, XXH_alignment align)
-{
- if (align==XXH_unaligned) {
- return XXH_readLE32(ptr);
- } else {
- return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr);
- }
-}
-
-
-/* *************************************
-* Misc
-***************************************/
-/*! @ingroup public */
-XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
-
-
-/* *******************************************************************
-* 32-bit hash functions
-*********************************************************************/
-/*!
- * @}
- * @defgroup xxh32_impl XXH32 implementation
- * @ingroup impl
- * @{
- */
- /* #define instead of static const, to be used as initializers */
-#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */
-#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */
-#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */
-#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */
-#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */
-
-#ifdef XXH_OLD_NAMES
-# define PRIME32_1 XXH_PRIME32_1
-# define PRIME32_2 XXH_PRIME32_2
-# define PRIME32_3 XXH_PRIME32_3
-# define PRIME32_4 XXH_PRIME32_4
-# define PRIME32_5 XXH_PRIME32_5
-#endif
-
-/*!
- * @internal
- * @brief Normal stripe processing routine.
- *
- * This shuffles the bits so that any bit from @p input impacts several bits in
- * @p acc.
- *
- * @param acc The accumulator lane.
- * @param input The stripe of input to mix.
- * @return The mixed accumulator lane.
- */
-static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input)
-{
- acc += input * XXH_PRIME32_2;
- acc = XXH_rotl32(acc, 13);
- acc *= XXH_PRIME32_1;
-#if (defined(__SSE4_1__) || defined(__aarch64__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
- /*
- * UGLY HACK:
- * A compiler fence is the only thing that prevents GCC and Clang from
- * autovectorizing the XXH32 loop (pragmas and attributes don't work for some
- * reason) without globally disabling SSE4.1.
- *
- * The reason we want to avoid vectorization is because despite working on
- * 4 integers at a time, there are multiple factors slowing XXH32 down on
- * SSE4:
- * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on
- * newer chips!) making it slightly slower to multiply four integers at
- * once compared to four integers independently. Even when pmulld was
- * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE
- * just to multiply unless doing a long operation.
- *
- * - Four instructions are required to rotate,
- * movqda tmp, v // not required with VEX encoding
- * pslld tmp, 13 // tmp <<= 13
- * psrld v, 19 // x >>= 19
- * por v, tmp // x |= tmp
- * compared to one for scalar:
- * roll v, 13 // reliably fast across the board
- * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason
- *
- * - Instruction level parallelism is actually more beneficial here because
- * the SIMD actually serializes this operation: While v1 is rotating, v2
- * can load data, while v3 can multiply. SSE forces them to operate
- * together.
- *
- * This is also enabled on AArch64, as Clang autovectorizes it incorrectly
- * and it is pointless writing a NEON implementation that is basically the
- * same speed as scalar for XXH32.
- */
- XXH_COMPILER_GUARD(acc);
-#endif
- return acc;
-}
-
-/*!
- * @internal
- * @brief Mixes all bits to finalize the hash.
- *
- * The final mix ensures that all input bits have a chance to impact any bit in
- * the output digest, resulting in an unbiased distribution.
- *
- * @param h32 The hash to avalanche.
- * @return The avalanched hash.
- */
-static xxh_u32 XXH32_avalanche(xxh_u32 h32)
-{
- h32 ^= h32 >> 15;
- h32 *= XXH_PRIME32_2;
- h32 ^= h32 >> 13;
- h32 *= XXH_PRIME32_3;
- h32 ^= h32 >> 16;
- return(h32);
-}
-
-#define XXH_get32bits(p) XXH_readLE32_align(p, align)
-
-/*!
- * @internal
- * @brief Processes the last 0-15 bytes of @p ptr.
- *
- * There may be up to 15 bytes remaining to consume from the input.
- * This final stage will digest them to ensure that all input bytes are present
- * in the final mix.
- *
- * @param h32 The hash to finalize.
- * @param ptr The pointer to the remaining input.
- * @param len The remaining length, modulo 16.
- * @param align Whether @p ptr is aligned.
- * @return The finalized hash.
- */
-static xxh_u32
-XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align)
-{
-#define XXH_PROCESS1 do { \
- h32 += (*ptr++) * XXH_PRIME32_5; \
- h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \
-} while (0)
-
-#define XXH_PROCESS4 do { \
- h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \
- ptr += 4; \
- h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \
-} while (0)
-
- if (ptr==NULL) XXH_ASSERT(len == 0);
-
- /* Compact rerolled version; generally faster */
- if (!XXH32_ENDJMP) {
- len &= 15;
- while (len >= 4) {
- XXH_PROCESS4;
- len -= 4;
- }
- while (len > 0) {
- XXH_PROCESS1;
- --len;
- }
- return XXH32_avalanche(h32);
- } else {
- switch(len&15) /* or switch(bEnd - p) */ {
- case 12: XXH_PROCESS4;
- XXH_FALLTHROUGH;
- case 8: XXH_PROCESS4;
- XXH_FALLTHROUGH;
- case 4: XXH_PROCESS4;
- return XXH32_avalanche(h32);
-
- case 13: XXH_PROCESS4;
- XXH_FALLTHROUGH;
- case 9: XXH_PROCESS4;
- XXH_FALLTHROUGH;
- case 5: XXH_PROCESS4;
- XXH_PROCESS1;
- return XXH32_avalanche(h32);
-
- case 14: XXH_PROCESS4;
- XXH_FALLTHROUGH;
- case 10: XXH_PROCESS4;
- XXH_FALLTHROUGH;
- case 6: XXH_PROCESS4;
- XXH_PROCESS1;
- XXH_PROCESS1;
- return XXH32_avalanche(h32);
-
- case 15: XXH_PROCESS4;
- XXH_FALLTHROUGH;
- case 11: XXH_PROCESS4;
- XXH_FALLTHROUGH;
- case 7: XXH_PROCESS4;
- XXH_FALLTHROUGH;
- case 3: XXH_PROCESS1;
- XXH_FALLTHROUGH;
- case 2: XXH_PROCESS1;
- XXH_FALLTHROUGH;
- case 1: XXH_PROCESS1;
- XXH_FALLTHROUGH;
- case 0: return XXH32_avalanche(h32);
- }
- XXH_ASSERT(0);
- return h32; /* reaching this point is deemed impossible */
- }
-}
-
-#ifdef XXH_OLD_NAMES
-# define PROCESS1 XXH_PROCESS1
-# define PROCESS4 XXH_PROCESS4
-#else
-# undef XXH_PROCESS1
-# undef XXH_PROCESS4
-#endif
-
-/*!
- * @internal
- * @brief The implementation for @ref XXH32().
- *
- * @param input , len , seed Directly passed from @ref XXH32().
- * @param align Whether @p input is aligned.
- * @return The calculated hash.
- */
-XXH_FORCE_INLINE xxh_u32
-XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
-{
- xxh_u32 h32;
-
- if (input==NULL) XXH_ASSERT(len == 0);
-
- if (len>=16) {
- const xxh_u8* const bEnd = input + len;
- const xxh_u8* const limit = bEnd - 15;
- xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
- xxh_u32 v2 = seed + XXH_PRIME32_2;
- xxh_u32 v3 = seed + 0;
- xxh_u32 v4 = seed - XXH_PRIME32_1;
-
- do {
- v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4;
- v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4;
- v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4;
- v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4;
- } while (input < limit);
-
- h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7)
- + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
- } else {
- h32 = seed + XXH_PRIME32_5;
- }
-
- h32 += (xxh_u32)len;
-
- return XXH32_finalize(h32, input, len&15, align);
-}
-
-/*! @ingroup xxh32_family */
-XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
-{
-#if 0
- /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
- XXH32_state_t state;
- XXH32_reset(&state, seed);
- XXH32_update(&state, (const xxh_u8*)input, len);
- return XXH32_digest(&state);
-#else
- if (XXH_FORCE_ALIGN_CHECK) {
- if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
- return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
- } }
-
- return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
-#endif
-}
-
-
-
-/******* Hash streaming *******/
-/*!
- * @ingroup xxh32_family
- */
-XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
-{
- return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
-}
-/*! @ingroup xxh32_family */
-XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
-{
- XXH_free(statePtr);
- return XXH_OK;
-}
-
-/*! @ingroup xxh32_family */
-XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
-{
- XXH_memcpy(dstState, srcState, sizeof(*dstState));
-}
-
-/*! @ingroup xxh32_family */
-XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
-{
- XXH_ASSERT(statePtr != NULL);
- memset(statePtr, 0, sizeof(*statePtr));
- statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
- statePtr->v[1] = seed + XXH_PRIME32_2;
- statePtr->v[2] = seed + 0;
- statePtr->v[3] = seed - XXH_PRIME32_1;
- return XXH_OK;
-}
-
-
-/*! @ingroup xxh32_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH32_update(XXH32_state_t* state, const void* input, size_t len)
-{
- if (input==NULL) {
- XXH_ASSERT(len == 0);
- return XXH_OK;
- }
-
- { const xxh_u8* p = (const xxh_u8*)input;
- const xxh_u8* const bEnd = p + len;
-
- state->total_len_32 += (XXH32_hash_t)len;
- state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
-
- if (state->memsize + len < 16) { /* fill in tmp buffer */
- XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len);
- state->memsize += (XXH32_hash_t)len;
- return XXH_OK;
- }
-
- if (state->memsize) { /* some data left from previous update */
- XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize);
- { const xxh_u32* p32 = state->mem32;
- state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++;
- state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++;
- state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++;
- state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32));
- }
- p += 16-state->memsize;
- state->memsize = 0;
- }
-
- if (p <= bEnd-16) {
- const xxh_u8* const limit = bEnd - 16;
-
- do {
- state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4;
- state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4;
- state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4;
- state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4;
- } while (p<=limit);
-
- }
-
- if (p < bEnd) {
- XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
- state->memsize = (unsigned)(bEnd-p);
- }
- }
-
- return XXH_OK;
-}
-
-
-/*! @ingroup xxh32_family */
-XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state)
-{
- xxh_u32 h32;
-
- if (state->large_len) {
- h32 = XXH_rotl32(state->v[0], 1)
- + XXH_rotl32(state->v[1], 7)
- + XXH_rotl32(state->v[2], 12)
- + XXH_rotl32(state->v[3], 18);
- } else {
- h32 = state->v[2] /* == seed */ + XXH_PRIME32_5;
- }
-
- h32 += state->total_len_32;
-
- return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned);
-}
-
-
-/******* Canonical representation *******/
-
-/*!
- * @ingroup xxh32_family
- * The default return values from XXH functions are unsigned 32 and 64 bit
- * integers.
- *
- * The canonical representation uses big endian convention, the same convention
- * as human-readable numbers (large digits first).
- *
- * This way, hash values can be written into a file or buffer, remaining
- * comparable across different systems.
- *
- * The following functions allow transformation of hash values to and from their
- * canonical format.
- */
-XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
-{
- /* XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); */
- if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
- XXH_memcpy(dst, &hash, sizeof(*dst));
-}
-/*! @ingroup xxh32_family */
-XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
-{
- return XXH_readBE32(src);
-}
-
-
-#ifndef XXH_NO_LONG_LONG
-
-/* *******************************************************************
-* 64-bit hash functions
-*********************************************************************/
-/*!
- * @}
- * @ingroup impl
- * @{
- */
-/******* Memory access *******/
-
-typedef XXH64_hash_t xxh_u64;
-
-#ifdef XXH_OLD_NAMES
-# define U64 xxh_u64
-#endif
-
-#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
-/*
- * Manual byteshift. Best for old compilers which don't inline memcpy.
- * We actually directly use XXH_readLE64 and XXH_readBE64.
- */
-#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
-
-/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
-static xxh_u64 XXH_read64(const void* memPtr)
-{
- return *(const xxh_u64*) memPtr;
-}
-
-#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
-
-/*
- * __pack instructions are safer, but compiler specific, hence potentially
- * problematic for some compilers.
- *
- * Currently only defined for GCC and ICC.
- */
-#ifdef XXH_OLD_NAMES
-typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64;
-#endif
-static xxh_u64 XXH_read64(const void* ptr)
-{
- typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64;
- return ((const xxh_unalign64*)ptr)->u64;
-}
-
-#else
-
-/*
- * Portable and safe solution. Generally efficient.
- * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
- */
-static xxh_u64 XXH_read64(const void* memPtr)
-{
- xxh_u64 val;
- XXH_memcpy(&val, memPtr, sizeof(val));
- return val;
-}
-
-#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
-
-#if defined(_MSC_VER) /* Visual Studio */
-# define XXH_swap64 _byteswap_uint64
-#elif XXH_GCC_VERSION >= 403
-# define XXH_swap64 __builtin_bswap64
-#else
-static xxh_u64 XXH_swap64(xxh_u64 x)
-{
- return ((x << 56) & 0xff00000000000000ULL) |
- ((x << 40) & 0x00ff000000000000ULL) |
- ((x << 24) & 0x0000ff0000000000ULL) |
- ((x << 8) & 0x000000ff00000000ULL) |
- ((x >> 8) & 0x00000000ff000000ULL) |
- ((x >> 24) & 0x0000000000ff0000ULL) |
- ((x >> 40) & 0x000000000000ff00ULL) |
- ((x >> 56) & 0x00000000000000ffULL);
-}
-#endif
-
-
-/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */
-#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
-
-XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr)
-{
- const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
- return bytePtr[0]
- | ((xxh_u64)bytePtr[1] << 8)
- | ((xxh_u64)bytePtr[2] << 16)
- | ((xxh_u64)bytePtr[3] << 24)
- | ((xxh_u64)bytePtr[4] << 32)
- | ((xxh_u64)bytePtr[5] << 40)
- | ((xxh_u64)bytePtr[6] << 48)
- | ((xxh_u64)bytePtr[7] << 56);
-}
-
-XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr)
-{
- const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
- return bytePtr[7]
- | ((xxh_u64)bytePtr[6] << 8)
- | ((xxh_u64)bytePtr[5] << 16)
- | ((xxh_u64)bytePtr[4] << 24)
- | ((xxh_u64)bytePtr[3] << 32)
- | ((xxh_u64)bytePtr[2] << 40)
- | ((xxh_u64)bytePtr[1] << 48)
- | ((xxh_u64)bytePtr[0] << 56);
-}
-
-#else
-XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr)
-{
- return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
-}
-
-static xxh_u64 XXH_readBE64(const void* ptr)
-{
- return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
-}
-#endif
-
-XXH_FORCE_INLINE xxh_u64
-XXH_readLE64_align(const void* ptr, XXH_alignment align)
-{
- if (align==XXH_unaligned)
- return XXH_readLE64(ptr);
- else
- return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr);
-}
-
-
-/******* xxh64 *******/
-/*!
- * @}
- * @defgroup xxh64_impl XXH64 implementation
- * @ingroup impl
- * @{
- */
-/* #define rather that static const, to be used as initializers */
-#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */
-#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */
-#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */
-#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */
-#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */
-
-#ifdef XXH_OLD_NAMES
-# define PRIME64_1 XXH_PRIME64_1
-# define PRIME64_2 XXH_PRIME64_2
-# define PRIME64_3 XXH_PRIME64_3
-# define PRIME64_4 XXH_PRIME64_4
-# define PRIME64_5 XXH_PRIME64_5
-#endif
-
-static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input)
-{
- acc += input * XXH_PRIME64_2;
- acc = XXH_rotl64(acc, 31);
- acc *= XXH_PRIME64_1;
- return acc;
-}
-
-static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val)
-{
- val = XXH64_round(0, val);
- acc ^= val;
- acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
- return acc;
-}
-
-static xxh_u64 XXH64_avalanche(xxh_u64 h64)
-{
- h64 ^= h64 >> 33;
- h64 *= XXH_PRIME64_2;
- h64 ^= h64 >> 29;
- h64 *= XXH_PRIME64_3;
- h64 ^= h64 >> 32;
- return h64;
-}
-
-
-#define XXH_get64bits(p) XXH_readLE64_align(p, align)
-
-static xxh_u64
-XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align)
-{
- if (ptr==NULL) XXH_ASSERT(len == 0);
- len &= 31;
- while (len >= 8) {
- xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr));
- ptr += 8;
- h64 ^= k1;
- h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4;
- len -= 8;
- }
- if (len >= 4) {
- h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1;
- ptr += 4;
- h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
- len -= 4;
- }
- while (len > 0) {
- h64 ^= (*ptr++) * XXH_PRIME64_5;
- h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1;
- --len;
- }
- return XXH64_avalanche(h64);
-}
-
-#ifdef XXH_OLD_NAMES
-# define PROCESS1_64 XXH_PROCESS1_64
-# define PROCESS4_64 XXH_PROCESS4_64
-# define PROCESS8_64 XXH_PROCESS8_64
-#else
-# undef XXH_PROCESS1_64
-# undef XXH_PROCESS4_64
-# undef XXH_PROCESS8_64
-#endif
-
-XXH_FORCE_INLINE xxh_u64
-XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align)
-{
- xxh_u64 h64;
- if (input==NULL) XXH_ASSERT(len == 0);
-
- if (len>=32) {
- const xxh_u8* const bEnd = input + len;
- const xxh_u8* const limit = bEnd - 31;
- xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
- xxh_u64 v2 = seed + XXH_PRIME64_2;
- xxh_u64 v3 = seed + 0;
- xxh_u64 v4 = seed - XXH_PRIME64_1;
-
- do {
- v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8;
- v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8;
- v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8;
- v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8;
- } while (input<limit);
-
- h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
- h64 = XXH64_mergeRound(h64, v1);
- h64 = XXH64_mergeRound(h64, v2);
- h64 = XXH64_mergeRound(h64, v3);
- h64 = XXH64_mergeRound(h64, v4);
-
- } else {
- h64 = seed + XXH_PRIME64_5;
- }
-
- h64 += (xxh_u64) len;
-
- return XXH64_finalize(h64, input, len, align);
-}
-
-
-/*! @ingroup xxh64_family */
-XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t len, XXH64_hash_t seed)
-{
-#if 0
- /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
- XXH64_state_t state;
- XXH64_reset(&state, seed);
- XXH64_update(&state, (const xxh_u8*)input, len);
- return XXH64_digest(&state);
-#else
- if (XXH_FORCE_ALIGN_CHECK) {
- if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
- return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
- } }
-
- return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
-
-#endif
-}
-
-/******* Hash Streaming *******/
-
-/*! @ingroup xxh64_family*/
-XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
-{
- return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
-}
-/*! @ingroup xxh64_family */
-XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
-{
- XXH_free(statePtr);
- return XXH_OK;
-}
-
-/*! @ingroup xxh64_family */
-XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
-{
- XXH_memcpy(dstState, srcState, sizeof(*dstState));
-}
-
-/*! @ingroup xxh64_family */
-XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed)
-{
- XXH_ASSERT(statePtr != NULL);
- memset(statePtr, 0, sizeof(*statePtr));
- statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
- statePtr->v[1] = seed + XXH_PRIME64_2;
- statePtr->v[2] = seed + 0;
- statePtr->v[3] = seed - XXH_PRIME64_1;
- return XXH_OK;
-}
-
-/*! @ingroup xxh64_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH64_update (XXH64_state_t* state, const void* input, size_t len)
-{
- if (input==NULL) {
- XXH_ASSERT(len == 0);
- return XXH_OK;
- }
-
- { const xxh_u8* p = (const xxh_u8*)input;
- const xxh_u8* const bEnd = p + len;
-
- state->total_len += len;
-
- if (state->memsize + len < 32) { /* fill in tmp buffer */
- XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len);
- state->memsize += (xxh_u32)len;
- return XXH_OK;
- }
-
- if (state->memsize) { /* tmp buffer is full */
- XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize);
- state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0));
- state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1));
- state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2));
- state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3));
- p += 32 - state->memsize;
- state->memsize = 0;
- }
-
- if (p+32 <= bEnd) {
- const xxh_u8* const limit = bEnd - 32;
-
- do {
- state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8;
- state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8;
- state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8;
- state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8;
- } while (p<=limit);
-
- }
-
- if (p < bEnd) {
- XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
- state->memsize = (unsigned)(bEnd-p);
- }
- }
-
- return XXH_OK;
-}
-
-
-/*! @ingroup xxh64_family */
-XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state)
-{
- xxh_u64 h64;
-
- if (state->total_len >= 32) {
- h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18);
- h64 = XXH64_mergeRound(h64, state->v[0]);
- h64 = XXH64_mergeRound(h64, state->v[1]);
- h64 = XXH64_mergeRound(h64, state->v[2]);
- h64 = XXH64_mergeRound(h64, state->v[3]);
- } else {
- h64 = state->v[2] /*seed*/ + XXH_PRIME64_5;
- }
-
- h64 += (xxh_u64) state->total_len;
-
- return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned);
-}
-
-
-/******* Canonical representation *******/
-
-/*! @ingroup xxh64_family */
-XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
-{
- /* XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); */
- if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
- XXH_memcpy(dst, &hash, sizeof(*dst));
-}
-
-/*! @ingroup xxh64_family */
-XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
-{
- return XXH_readBE64(src);
-}
-
-#ifndef XXH_NO_XXH3
-
-/* *********************************************************************
-* XXH3
-* New generation hash designed for speed on small keys and vectorization
-************************************************************************ */
-/*!
- * @}
- * @defgroup xxh3_impl XXH3 implementation
- * @ingroup impl
- * @{
- */
-
-/* === Compiler specifics === */
-
-#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
-# define XXH_RESTRICT /* disable */
-#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */
-# define XXH_RESTRICT restrict
-#else
-/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */
-# define XXH_RESTRICT /* disable */
-#endif
-
-#if (defined(__GNUC__) && (__GNUC__ >= 3)) \
- || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
- || defined(__clang__)
-# define XXH_likely(x) __builtin_expect(x, 1)
-# define XXH_unlikely(x) __builtin_expect(x, 0)
-#else
-# define XXH_likely(x) (x)
-# define XXH_unlikely(x) (x)
-#endif
-
-#if defined(__GNUC__) || defined(__clang__)
-# if defined(__ARM_NEON__) || defined(__ARM_NEON) \
- || defined(__aarch64__) || defined(_M_ARM) \
- || defined(_M_ARM64) || defined(_M_ARM64EC)
-# define inline __inline__ /* circumvent a clang bug */
-# include <arm_neon.h>
-# undef inline
-# elif defined(__AVX2__)
-# include <immintrin.h>
-# elif defined(__SSE2__)
-# include <emmintrin.h>
-# endif
-#endif
-
-#if defined(_MSC_VER)
-# include <intrin.h>
-#endif
-
-/*
- * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while
- * remaining a true 64-bit/128-bit hash function.
- *
- * This is done by prioritizing a subset of 64-bit operations that can be
- * emulated without too many steps on the average 32-bit machine.
- *
- * For example, these two lines seem similar, and run equally fast on 64-bit:
- *
- * xxh_u64 x;
- * x ^= (x >> 47); // good
- * x ^= (x >> 13); // bad
- *
- * However, to a 32-bit machine, there is a major difference.
- *
- * x ^= (x >> 47) looks like this:
- *
- * x.lo ^= (x.hi >> (47 - 32));
- *
- * while x ^= (x >> 13) looks like this:
- *
- * // note: funnel shifts are not usually cheap.
- * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13));
- * x.hi ^= (x.hi >> 13);
- *
- * The first one is significantly faster than the second, simply because the
- * shift is larger than 32. This means:
- * - All the bits we need are in the upper 32 bits, so we can ignore the lower
- * 32 bits in the shift.
- * - The shift result will always fit in the lower 32 bits, and therefore,
- * we can ignore the upper 32 bits in the xor.
- *
- * Thanks to this optimization, XXH3 only requires these features to be efficient:
- *
- * - Usable unaligned access
- * - A 32-bit or 64-bit ALU
- * - If 32-bit, a decent ADC instruction
- * - A 32 or 64-bit multiply with a 64-bit result
- * - For the 128-bit variant, a decent byteswap helps short inputs.
- *
- * The first two are already required by XXH32, and almost all 32-bit and 64-bit
- * platforms which can run XXH32 can run XXH3 efficiently.
- *
- * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one
- * notable exception.
- *
- * First of all, Thumb-1 lacks support for the UMULL instruction which
- * performs the important long multiply. This means numerous __aeabi_lmul
- * calls.
- *
- * Second of all, the 8 functional registers are just not enough.
- * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
- * Lo registers, and this shuffling results in thousands more MOVs than A32.
- *
- * A32 and T32 don't have this limitation. They can access all 14 registers,
- * do a 32->64 multiply with UMULL, and the flexible operand allowing free
- * shifts is helpful, too.
- *
- * Therefore, we do a quick sanity check.
- *
- * If compiling Thumb-1 for a target which supports ARM instructions, we will
- * emit a warning, as it is not a "sane" platform to compile for.
- *
- * Usually, if this happens, it is because of an accident and you probably need
- * to specify -march, as you likely meant to compile for a newer architecture.
- *
- * Credit: large sections of the vectorial and asm source code paths
- * have been contributed by @easyaspi314
- */
-#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
-# warning "XXH3 is highly inefficient without ARM or Thumb-2."
-#endif
-
-/* ==========================================
- * Vectorization detection
- * ========================================== */
-
-#ifdef XXH_DOXYGEN
-/*!
- * @ingroup tuning
- * @brief Overrides the vectorization implementation chosen for XXH3.
- *
- * Can be defined to 0 to disable SIMD or any of the values mentioned in
- * @ref XXH_VECTOR_TYPE.
- *
- * If this is not defined, it uses predefined macros to determine the best
- * implementation.
- */
-# define XXH_VECTOR XXH_SCALAR
-/*!
- * @ingroup tuning
- * @brief Possible values for @ref XXH_VECTOR.
- *
- * Note that these are actually implemented as macros.
- *
- * If this is not defined, it is detected automatically.
- * @ref XXH_X86DISPATCH overrides this.
- */
-enum XXH_VECTOR_TYPE /* fake enum */ {
- XXH_SCALAR = 0, /*!< Portable scalar version */
- XXH_SSE2 = 1, /*!<
- * SSE2 for Pentium 4, Opteron, all x86_64.
- *
- * @note SSE2 is also guaranteed on Windows 10, macOS, and
- * Android x86.
- */
- XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */
- XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */
- XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */
- XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */
-};
-/*!
- * @ingroup tuning
- * @brief Selects the minimum alignment for XXH3's accumulators.
- *
- * When using SIMD, this should match the alignment reqired for said vector
- * type, so, for example, 32 for AVX2.
- *
- * Default: Auto detected.
- */
-# define XXH_ACC_ALIGN 8
-#endif
-
-/* Actual definition */
-#ifndef XXH_DOXYGEN
-# define XXH_SCALAR 0
-# define XXH_SSE2 1
-# define XXH_AVX2 2
-# define XXH_AVX512 3
-# define XXH_NEON 4
-# define XXH_VSX 5
-#endif
-
-#ifndef XXH_VECTOR /* can be defined on command line */
-# if ( \
- defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \
- || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \
- ) && ( \
- defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \
- || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
- )
-# define XXH_VECTOR XXH_NEON
-# elif defined(__AVX512F__)
-# define XXH_VECTOR XXH_AVX512
-# elif defined(__AVX2__)
-# define XXH_VECTOR XXH_AVX2
-# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
-# define XXH_VECTOR XXH_SSE2
-# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
- || (defined(__s390x__) && defined(__VEC__)) \
- && defined(__GNUC__) /* TODO: IBM XL */
-# define XXH_VECTOR XXH_VSX
-# else
-# define XXH_VECTOR XXH_SCALAR
-# endif
-#endif
-
-/*
- * Controls the alignment of the accumulator,
- * for compatibility with aligned vector loads, which are usually faster.
- */
-#ifndef XXH_ACC_ALIGN
-# if defined(XXH_X86DISPATCH)
-# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */
-# elif XXH_VECTOR == XXH_SCALAR /* scalar */
-# define XXH_ACC_ALIGN 8
-# elif XXH_VECTOR == XXH_SSE2 /* sse2 */
-# define XXH_ACC_ALIGN 16
-# elif XXH_VECTOR == XXH_AVX2 /* avx2 */
-# define XXH_ACC_ALIGN 32
-# elif XXH_VECTOR == XXH_NEON /* neon */
-# define XXH_ACC_ALIGN 16
-# elif XXH_VECTOR == XXH_VSX /* vsx */
-# define XXH_ACC_ALIGN 16
-# elif XXH_VECTOR == XXH_AVX512 /* avx512 */
-# define XXH_ACC_ALIGN 64
-# endif
-#endif
-
-#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \
- || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512
-# define XXH_SEC_ALIGN XXH_ACC_ALIGN
-#else
-# define XXH_SEC_ALIGN 8
-#endif
-
-/*
- * UGLY HACK:
- * GCC usually generates the best code with -O3 for xxHash.
- *
- * However, when targeting AVX2, it is overzealous in its unrolling resulting
- * in code roughly 3/4 the speed of Clang.
- *
- * There are other issues, such as GCC splitting _mm256_loadu_si256 into
- * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which
- * only applies to Sandy and Ivy Bridge... which don't even support AVX2.
- *
- * That is why when compiling the AVX2 version, it is recommended to use either
- * -O2 -mavx2 -march=haswell
- * or
- * -O2 -mavx2 -mno-avx256-split-unaligned-load
- * for decent performance, or to use Clang instead.
- *
- * Fortunately, we can control the first one with a pragma that forces GCC into
- * -O2, but the other one we can't control without "failed to inline always
- * inline function due to target mismatch" warnings.
- */
-#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
- && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
- && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */
-# pragma GCC push_options
-# pragma GCC optimize("-O2")
-#endif
-
-
-#if XXH_VECTOR == XXH_NEON
-/*
- * NEON's setup for vmlal_u32 is a little more complicated than it is on
- * SSE2, AVX2, and VSX.
- *
- * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast.
- *
- * To do the same operation, the 128-bit 'Q' register needs to be split into
- * two 64-bit 'D' registers, performing this operation::
- *
- * [ a | b ]
- * | '---------. .--------' |
- * | x |
- * | .---------' '--------. |
- * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ]
- *
- * Due to significant changes in aarch64, the fastest method for aarch64 is
- * completely different than the fastest method for ARMv7-A.
- *
- * ARMv7-A treats D registers as unions overlaying Q registers, so modifying
- * D11 will modify the high half of Q5. This is similar to how modifying AH
- * will only affect bits 8-15 of AX on x86.
- *
- * VZIP takes two registers, and puts even lanes in one register and odd lanes
- * in the other.
- *
- * On ARMv7-A, this strangely modifies both parameters in place instead of
- * taking the usual 3-operand form.
- *
- * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the
- * lower and upper halves of the Q register to end up with the high and low
- * halves where we want - all in one instruction.
- *
- * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] }
- *
- * Unfortunately we need inline assembly for this: Instructions modifying two
- * registers at once is not possible in GCC or Clang's IR, and they have to
- * create a copy.
- *
- * aarch64 requires a different approach.
- *
- * In order to make it easier to write a decent compiler for aarch64, many
- * quirks were removed, such as conditional execution.
- *
- * NEON was also affected by this.
- *
- * aarch64 cannot access the high bits of a Q-form register, and writes to a
- * D-form register zero the high bits, similar to how writes to W-form scalar
- * registers (or DWORD registers on x86_64) work.
- *
- * The formerly free vget_high intrinsics now require a vext (with a few
- * exceptions)
- *
- * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent
- * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one
- * operand.
- *
- * The equivalent of the VZIP.32 on the lower and upper halves would be this
- * mess:
- *
- * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] }
- * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] }
- * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] }
- *
- * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN):
- *
- * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32);
- * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF);
- *
- * This is available on ARMv7-A, but is less efficient than a single VZIP.32.
- */
-
-/*!
- * Function-like macro:
- * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi)
- * {
- * outLo = (uint32x2_t)(in & 0xFFFFFFFF);
- * outHi = (uint32x2_t)(in >> 32);
- * in = UNDEFINED;
- * }
- */
-# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \
- && (defined(__GNUC__) || defined(__clang__)) \
- && (defined(__arm__) || defined(__thumb__) || defined(_M_ARM))
-# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \
- do { \
- /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \
- /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \
- /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \
- __asm__("vzip.32 %e0, %f0" : "+w" (in)); \
- (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \
- (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \
- } while (0)
-# else
-# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \
- do { \
- (outLo) = vmovn_u64 (in); \
- (outHi) = vshrn_n_u64 ((in), 32); \
- } while (0)
-# endif
-
-/*!
- * @ingroup tuning
- * @brief Controls the NEON to scalar ratio for XXH3
- *
- * On AArch64 when not optimizing for size, XXH3 will run 6 lanes using NEON and
- * 2 lanes on scalar by default.
- *
- * This can be set to 2, 4, 6, or 8. ARMv7 will default to all 8 NEON lanes, as the
- * emulated 64-bit arithmetic is too slow.
- *
- * Modern ARM CPUs are _very_ sensitive to how their pipelines are used.
- *
- * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but it can't
- * have more than 2 NEON (F0/F1) micro-ops. If you are only using NEON instructions,
- * you are only using 2/3 of the CPU bandwidth.
- *
- * This is even more noticable on the more advanced cores like the A76 which
- * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once.
- *
- * Therefore, @ref XXH3_NEON_LANES lanes will be processed using NEON, and the
- * remaining lanes will use scalar instructions. This improves the bandwidth
- * and also gives the integer pipelines something to do besides twiddling loop
- * counters and pointers.
- *
- * This change benefits CPUs with large micro-op buffers without negatively affecting
- * other CPUs:
- *
- * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. |
- * |:----------------------|:--------------------|----------:|-----------:|------:|
- * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% |
- * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% |
- * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% |
- *
- * It also seems to fix some bad codegen on GCC, making it almost as fast as clang.
- *
- * @see XXH3_accumulate_512_neon()
- */
-# ifndef XXH3_NEON_LANES
-# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \
- && !defined(__OPTIMIZE_SIZE__)
-# define XXH3_NEON_LANES 6
-# else
-# define XXH3_NEON_LANES XXH_ACC_NB
-# endif
-# endif
-#endif /* XXH_VECTOR == XXH_NEON */
-
-/*
- * VSX and Z Vector helpers.
- *
- * This is very messy, and any pull requests to clean this up are welcome.
- *
- * There are a lot of problems with supporting VSX and s390x, due to
- * inconsistent intrinsics, spotty coverage, and multiple endiannesses.
- */
-#if XXH_VECTOR == XXH_VSX
-# if defined(__s390x__)
-# include <s390intrin.h>
-# else
-/* gcc's altivec.h can have the unwanted consequence to unconditionally
- * #define bool, vector, and pixel keywords,
- * with bad consequences for programs already using these keywords for other purposes.
- * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined.
- * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler,
- * but it seems that, in some cases, it isn't.
- * Force the build macro to be defined, so that keywords are not altered.
- */
-# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__)
-# define __APPLE_ALTIVEC__
-# endif
-# include <altivec.h>
-# endif
-
-typedef __vector unsigned long long xxh_u64x2;
-typedef __vector unsigned char xxh_u8x16;
-typedef __vector unsigned xxh_u32x4;
-
-# ifndef XXH_VSX_BE
-# if defined(__BIG_ENDIAN__) \
- || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
-# define XXH_VSX_BE 1
-# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
-# warning "-maltivec=be is not recommended. Please use native endianness."
-# define XXH_VSX_BE 1
-# else
-# define XXH_VSX_BE 0
-# endif
-# endif /* !defined(XXH_VSX_BE) */
-
-# if XXH_VSX_BE
-# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
-# define XXH_vec_revb vec_revb
-# else
-/*!
- * A polyfill for POWER9's vec_revb().
- */
-XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
-{
- xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
- 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
- return vec_perm(val, val, vByteSwap);
-}
-# endif
-# endif /* XXH_VSX_BE */
-
-/*!
- * Performs an unaligned vector load and byte swaps it on big endian.
- */
-XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr)
-{
- xxh_u64x2 ret;
- XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2));
-# if XXH_VSX_BE
- ret = XXH_vec_revb(ret);
-# endif
- return ret;
-}
-
-/*
- * vec_mulo and vec_mule are very problematic intrinsics on PowerPC
- *
- * These intrinsics weren't added until GCC 8, despite existing for a while,
- * and they are endian dependent. Also, their meaning swap depending on version.
- * */
-# if defined(__s390x__)
- /* s390x is always big endian, no issue on this platform */
-# define XXH_vec_mulo vec_mulo
-# define XXH_vec_mule vec_mule
-# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw)
-/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */
-# define XXH_vec_mulo __builtin_altivec_vmulouw
-# define XXH_vec_mule __builtin_altivec_vmuleuw
-# else
-/* gcc needs inline assembly */
-/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
-XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b)
-{
- xxh_u64x2 result;
- __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
- return result;
-}
-XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
-{
- xxh_u64x2 result;
- __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
- return result;
-}
-# endif /* XXH_vec_mulo, XXH_vec_mule */
-#endif /* XXH_VECTOR == XXH_VSX */
-
-
-/* prefetch
- * can be disabled, by declaring XXH_NO_PREFETCH build macro */
-#if defined(XXH_NO_PREFETCH)
-# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
-#else
-# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */
-# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
-# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
-# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
-# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
-# else
-# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
-# endif
-#endif /* XXH_NO_PREFETCH */
-
-
-/* ==========================================
- * XXH3 default settings
- * ========================================== */
-
-#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */
-
-#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
-# error "default keyset is not large enough"
-#endif
-
-/*! Pseudorandom secret taken directly from FARSH. */
-XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = {
- 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
- 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
- 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
- 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
- 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
- 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
- 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
- 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
- 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
- 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
- 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
- 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
-};
-
-
-#ifdef XXH_OLD_NAMES
-# define kSecret XXH3_kSecret
-#endif
-
-#ifdef XXH_DOXYGEN
-/*!
- * @brief Calculates a 32-bit to 64-bit long multiply.
- *
- * Implemented as a macro.
- *
- * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't
- * need to (but it shouldn't need to anyways, it is about 7 instructions to do
- * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we
- * use that instead of the normal method.
- *
- * If you are compiling for platforms like Thumb-1 and don't have a better option,
- * you may also want to write your own long multiply routine here.
- *
- * @param x, y Numbers to be multiplied
- * @return 64-bit product of the low 32 bits of @p x and @p y.
- */
-XXH_FORCE_INLINE xxh_u64
-XXH_mult32to64(xxh_u64 x, xxh_u64 y)
-{
- return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF);
-}
-#elif defined(_MSC_VER) && defined(_M_IX86)
-# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y))
-#else
-/*
- * Downcast + upcast is usually better than masking on older compilers like
- * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers.
- *
- * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands
- * and perform a full 64x64 multiply -- entirely redundant on 32-bit.
- */
-# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y))
-#endif
-
-/*!
- * @brief Calculates a 64->128-bit long multiply.
- *
- * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar
- * version.
- *
- * @param lhs , rhs The 64-bit integers to be multiplied
- * @return The 128-bit result represented in an @ref XXH128_hash_t.
- */
-static XXH128_hash_t
-XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
-{
- /*
- * GCC/Clang __uint128_t method.
- *
- * On most 64-bit targets, GCC and Clang define a __uint128_t type.
- * This is usually the best way as it usually uses a native long 64-bit
- * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
- *
- * Usually.
- *
- * Despite being a 32-bit platform, Clang (and emscripten) define this type
- * despite not having the arithmetic for it. This results in a laggy
- * compiler builtin call which calculates a full 128-bit multiply.
- * In that case it is best to use the portable one.
- * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
- */
-#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \
- && defined(__SIZEOF_INT128__) \
- || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
-
- __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs;
- XXH128_hash_t r128;
- r128.low64 = (xxh_u64)(product);
- r128.high64 = (xxh_u64)(product >> 64);
- return r128;
-
- /*
- * MSVC for x64's _umul128 method.
- *
- * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
- *
- * This compiles to single operand MUL on x64.
- */
-#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC)
-
-#ifndef _MSC_VER
-# pragma intrinsic(_umul128)
-#endif
- xxh_u64 product_high;
- xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
- XXH128_hash_t r128;
- r128.low64 = product_low;
- r128.high64 = product_high;
- return r128;
-
- /*
- * MSVC for ARM64's __umulh method.
- *
- * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method.
- */
-#elif defined(_M_ARM64) || defined(_M_ARM64EC)
-
-#ifndef _MSC_VER
-# pragma intrinsic(__umulh)
-#endif
- XXH128_hash_t r128;
- r128.low64 = lhs * rhs;
- r128.high64 = __umulh(lhs, rhs);
- return r128;
-
-#else
- /*
- * Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
- *
- * This is a fast and simple grade school multiply, which is shown below
- * with base 10 arithmetic instead of base 0x100000000.
- *
- * 9 3 // D2 lhs = 93
- * x 7 5 // D2 rhs = 75
- * ----------
- * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15
- * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45
- * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21
- * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63
- * ---------
- * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27
- * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67
- * ---------
- * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975
- *
- * The reasons for adding the products like this are:
- * 1. It avoids manual carry tracking. Just like how
- * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX.
- * This avoids a lot of complexity.
- *
- * 2. It hints for, and on Clang, compiles to, the powerful UMAAL
- * instruction available in ARM's Digital Signal Processing extension
- * in 32-bit ARMv6 and later, which is shown below:
- *
- * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm)
- * {
- * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm;
- * *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
- * *RdHi = (xxh_u32)(product >> 32);
- * }
- *
- * This instruction was designed for efficient long multiplication, and
- * allows this to be calculated in only 4 instructions at speeds
- * comparable to some 64-bit ALUs.
- *
- * 3. It isn't terrible on other platforms. Usually this will be a couple
- * of 32-bit ADD/ADCs.
- */
-
- /* First calculate all of the cross products. */
- xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
- xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF);
- xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
- xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32);
-
- /* Now add the products together. These will never overflow. */
- xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
- xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi;
- xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);
-
- XXH128_hash_t r128;
- r128.low64 = lower;
- r128.high64 = upper;
- return r128;
-#endif
-}
-
-/*!
- * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it.
- *
- * The reason for the separate function is to prevent passing too many structs
- * around by value. This will hopefully inline the multiply, but we don't force it.
- *
- * @param lhs , rhs The 64-bit integers to multiply
- * @return The low 64 bits of the product XOR'd by the high 64 bits.
- * @see XXH_mult64to128()
- */
-static xxh_u64
-XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
-{
- XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
- return product.low64 ^ product.high64;
-}
-
-/*! Seems to produce slightly better code on GCC for some reason. */
-XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift)
-{
- XXH_ASSERT(0 <= shift && shift < 64);
- return v64 ^ (v64 >> shift);
-}
-
-/*
- * This is a fast avalanche stage,
- * suitable when input bits are already partially mixed
- */
-static XXH64_hash_t XXH3_avalanche(xxh_u64 h64)
-{
- h64 = XXH_xorshift64(h64, 37);
- h64 *= 0x165667919E3779F9ULL;
- h64 = XXH_xorshift64(h64, 32);
- return h64;
-}
-
-/*
- * This is a stronger avalanche,
- * inspired by Pelle Evensen's rrmxmx
- * preferable when input has not been previously mixed
- */
-static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len)
-{
- /* this mix is inspired by Pelle Evensen's rrmxmx */
- h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24);
- h64 *= 0x9FB21C651E98DF25ULL;
- h64 ^= (h64 >> 35) + len ;
- h64 *= 0x9FB21C651E98DF25ULL;
- return XXH_xorshift64(h64, 28);
-}
-
-
-/* ==========================================
- * Short keys
- * ==========================================
- * One of the shortcomings of XXH32 and XXH64 was that their performance was
- * sub-optimal on short lengths. It used an iterative algorithm which strongly
- * favored lengths that were a multiple of 4 or 8.
- *
- * Instead of iterating over individual inputs, we use a set of single shot
- * functions which piece together a range of lengths and operate in constant time.
- *
- * Additionally, the number of multiplies has been significantly reduced. This
- * reduces latency, especially when emulating 64-bit multiplies on 32-bit.
- *
- * Depending on the platform, this may or may not be faster than XXH32, but it
- * is almost guaranteed to be faster than XXH64.
- */
-
-/*
- * At very short lengths, there isn't enough input to fully hide secrets, or use
- * the entire secret.
- *
- * There is also only a limited amount of mixing we can do before significantly
- * impacting performance.
- *
- * Therefore, we use different sections of the secret and always mix two secret
- * samples with an XOR. This should have no effect on performance on the
- * seedless or withSeed variants because everything _should_ be constant folded
- * by modern compilers.
- *
- * The XOR mixing hides individual parts of the secret and increases entropy.
- *
- * This adds an extra layer of strength for custom secrets.
- */
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
-{
- XXH_ASSERT(input != NULL);
- XXH_ASSERT(1 <= len && len <= 3);
- XXH_ASSERT(secret != NULL);
- /*
- * len = 1: combined = { input[0], 0x01, input[0], input[0] }
- * len = 2: combined = { input[1], 0x02, input[0], input[1] }
- * len = 3: combined = { input[2], 0x03, input[0], input[1] }
- */
- { xxh_u8 const c1 = input[0];
- xxh_u8 const c2 = input[len >> 1];
- xxh_u8 const c3 = input[len - 1];
- xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24)
- | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
- xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
- xxh_u64 const keyed = (xxh_u64)combined ^ bitflip;
- return XXH64_avalanche(keyed);
- }
-}
-
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
-{
- XXH_ASSERT(input != NULL);
- XXH_ASSERT(secret != NULL);
- XXH_ASSERT(4 <= len && len <= 8);
- seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
- { xxh_u32 const input1 = XXH_readLE32(input);
- xxh_u32 const input2 = XXH_readLE32(input + len - 4);
- xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed;
- xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32);
- xxh_u64 const keyed = input64 ^ bitflip;
- return XXH3_rrmxmx(keyed, len);
- }
-}
-
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
-{
- XXH_ASSERT(input != NULL);
- XXH_ASSERT(secret != NULL);
- XXH_ASSERT(9 <= len && len <= 16);
- { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed;
- xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed;
- xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1;
- xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2;
- xxh_u64 const acc = len
- + XXH_swap64(input_lo) + input_hi
- + XXH3_mul128_fold64(input_lo, input_hi);
- return XXH3_avalanche(acc);
- }
-}
-
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
-{
- XXH_ASSERT(len <= 16);
- { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed);
- if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed);
- if (len) return XXH3_len_1to3_64b(input, len, secret, seed);
- return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64)));
- }
-}
-
-/*
- * DISCLAIMER: There are known *seed-dependent* multicollisions here due to
- * multiplication by zero, affecting hashes of lengths 17 to 240.
- *
- * However, they are very unlikely.
- *
- * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
- * unseeded non-cryptographic hashes, it does not attempt to defend itself
- * against specially crafted inputs, only random inputs.
- *
- * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
- * cancelling out the secret is taken an arbitrary number of times (addressed
- * in XXH3_accumulate_512), this collision is very unlikely with random inputs
- * and/or proper seeding:
- *
- * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
- * function that is only called up to 16 times per hash with up to 240 bytes of
- * input.
- *
- * This is not too bad for a non-cryptographic hash function, especially with
- * only 64 bit outputs.
- *
- * The 128-bit variant (which trades some speed for strength) is NOT affected
- * by this, although it is always a good idea to use a proper seed if you care
- * about strength.
- */
-XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
- const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
-{
-#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
- && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \
- && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */
- /*
- * UGLY HACK:
- * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in
- * slower code.
- *
- * By forcing seed64 into a register, we disrupt the cost model and
- * cause it to scalarize. See `XXH32_round()`
- *
- * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600,
- * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on
- * GCC 9.2, despite both emitting scalar code.
- *
- * GCC generates much better scalar code than Clang for the rest of XXH3,
- * which is why finding a more optimal codepath is an interest.
- */
- XXH_COMPILER_GUARD(seed64);
-#endif
- { xxh_u64 const input_lo = XXH_readLE64(input);
- xxh_u64 const input_hi = XXH_readLE64(input+8);
- return XXH3_mul128_fold64(
- input_lo ^ (XXH_readLE64(secret) + seed64),
- input_hi ^ (XXH_readLE64(secret+8) - seed64)
- );
- }
-}
-
-/* For mid range keys, XXH3 uses a Mum-hash variant. */
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
- XXH64_hash_t seed)
-{
- XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
- XXH_ASSERT(16 < len && len <= 128);
-
- { xxh_u64 acc = len * XXH_PRIME64_1;
- if (len > 32) {
- if (len > 64) {
- if (len > 96) {
- acc += XXH3_mix16B(input+48, secret+96, seed);
- acc += XXH3_mix16B(input+len-64, secret+112, seed);
- }
- acc += XXH3_mix16B(input+32, secret+64, seed);
- acc += XXH3_mix16B(input+len-48, secret+80, seed);
- }
- acc += XXH3_mix16B(input+16, secret+32, seed);
- acc += XXH3_mix16B(input+len-32, secret+48, seed);
- }
- acc += XXH3_mix16B(input+0, secret+0, seed);
- acc += XXH3_mix16B(input+len-16, secret+16, seed);
-
- return XXH3_avalanche(acc);
- }
-}
-
-#define XXH3_MIDSIZE_MAX 240
-
-XXH_NO_INLINE XXH64_hash_t
-XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
- XXH64_hash_t seed)
-{
- XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
- XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
-
- #define XXH3_MIDSIZE_STARTOFFSET 3
- #define XXH3_MIDSIZE_LASTOFFSET 17
-
- { xxh_u64 acc = len * XXH_PRIME64_1;
- int const nbRounds = (int)len / 16;
- int i;
- for (i=0; i<8; i++) {
- acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed);
- }
- acc = XXH3_avalanche(acc);
- XXH_ASSERT(nbRounds >= 8);
-#if defined(__clang__) /* Clang */ \
- && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
- && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
- /*
- * UGLY HACK:
- * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86.
- * In everywhere else, it uses scalar code.
- *
- * For 64->128-bit multiplies, even if the NEON was 100% optimal, it
- * would still be slower than UMAAL (see XXH_mult64to128).
- *
- * Unfortunately, Clang doesn't handle the long multiplies properly and
- * converts them to the nonexistent "vmulq_u64" intrinsic, which is then
- * scalarized into an ugly mess of VMOV.32 instructions.
- *
- * This mess is difficult to avoid without turning autovectorization
- * off completely, but they are usually relatively minor and/or not
- * worth it to fix.
- *
- * This loop is the easiest to fix, as unlike XXH32, this pragma
- * _actually works_ because it is a loop vectorization instead of an
- * SLP vectorization.
- */
- #pragma clang loop vectorize(disable)
-#endif
- for (i=8 ; i < nbRounds; i++) {
- acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
- }
- /* last bytes */
- acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
- return XXH3_avalanche(acc);
- }
-}
-
-
-/* ======= Long Keys ======= */
-
-#define XXH_STRIPE_LEN 64
-#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */
-#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64))
-
-#ifdef XXH_OLD_NAMES
-# define STRIPE_LEN XXH_STRIPE_LEN
-# define ACC_NB XXH_ACC_NB
-#endif
-
-XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
-{
- if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
- XXH_memcpy(dst, &v64, sizeof(v64));
-}
-
-/* Several intrinsic functions below are supposed to accept __int64 as argument,
- * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ .
- * However, several environments do not define __int64 type,
- * requiring a workaround.
- */
-#if !defined (__VMS) \
- && (defined (__cplusplus) \
- || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
- typedef int64_t xxh_i64;
-#else
- /* the following type must have a width of 64-bit */
- typedef long long xxh_i64;
-#endif
-
-
-/*
- * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized.
- *
- * It is a hardened version of UMAC, based off of FARSH's implementation.
- *
- * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD
- * implementations, and it is ridiculously fast.
- *
- * We harden it by mixing the original input to the accumulators as well as the product.
- *
- * This means that in the (relatively likely) case of a multiply by zero, the
- * original input is preserved.
- *
- * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve
- * cross-pollination, as otherwise the upper and lower halves would be
- * essentially independent.
- *
- * This doesn't matter on 64-bit hashes since they all get merged together in
- * the end, so we skip the extra step.
- *
- * Both XXH3_64bits and XXH3_128bits use this subroutine.
- */
-
-#if (XXH_VECTOR == XXH_AVX512) \
- || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0)
-
-#ifndef XXH_TARGET_AVX512
-# define XXH_TARGET_AVX512 /* disable attribute target */
-#endif
-
-XXH_FORCE_INLINE XXH_TARGET_AVX512 void
-XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc,
- const void* XXH_RESTRICT input,
- const void* XXH_RESTRICT secret)
-{
- __m512i* const xacc = (__m512i *) acc;
- XXH_ASSERT((((size_t)acc) & 63) == 0);
- XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
-
- {
- /* data_vec = input[0]; */
- __m512i const data_vec = _mm512_loadu_si512 (input);
- /* key_vec = secret[0]; */
- __m512i const key_vec = _mm512_loadu_si512 (secret);
- /* data_key = data_vec ^ key_vec; */
- __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec);
- /* data_key_lo = data_key >> 32; */
- __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1));
- /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
- __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo);
- /* xacc[0] += swap(data_vec); */
- __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2));
- __m512i const sum = _mm512_add_epi64(*xacc, data_swap);
- /* xacc[0] += product; */
- *xacc = _mm512_add_epi64(product, sum);
- }
-}
-
-/*
- * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing.
- *
- * Multiplication isn't perfect, as explained by Google in HighwayHash:
- *
- * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
- * // varying degrees. In descending order of goodness, bytes
- * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
- * // As expected, the upper and lower bytes are much worse.
- *
- * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291
- *
- * Since our algorithm uses a pseudorandom secret to add some variance into the
- * mix, we don't need to (or want to) mix as often or as much as HighwayHash does.
- *
- * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid
- * extraction.
- *
- * Both XXH3_64bits and XXH3_128bits use this subroutine.
- */
-
-XXH_FORCE_INLINE XXH_TARGET_AVX512 void
-XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
-{
- XXH_ASSERT((((size_t)acc) & 63) == 0);
- XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
- { __m512i* const xacc = (__m512i*) acc;
- const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1);
-
- /* xacc[0] ^= (xacc[0] >> 47) */
- __m512i const acc_vec = *xacc;
- __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47);
- __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted);
- /* xacc[0] ^= secret; */
- __m512i const key_vec = _mm512_loadu_si512 (secret);
- __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec);
-
- /* xacc[0] *= XXH_PRIME32_1; */
- __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1));
- __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32);
- __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32);
- *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32));
- }
-}
-
-XXH_FORCE_INLINE XXH_TARGET_AVX512 void
-XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
-{
- XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0);
- XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64);
- XXH_ASSERT(((size_t)customSecret & 63) == 0);
- (void)(&XXH_writeLE64);
- { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
- __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64));
-
- const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret);
- __m512i* const dest = ( __m512i*) customSecret;
- int i;
- XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */
- XXH_ASSERT(((size_t)dest & 63) == 0);
- for (i=0; i < nbRounds; ++i) {
- /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*',
- * this will warn "discards 'const' qualifier". */
- union {
- const __m512i* cp;
- void* p;
- } remote_const_void;
- remote_const_void.cp = src + i;
- dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed);
- } }
-}
-
-#endif
-
-#if (XXH_VECTOR == XXH_AVX2) \
- || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0)
-
-#ifndef XXH_TARGET_AVX2
-# define XXH_TARGET_AVX2 /* disable attribute target */
-#endif
-
-XXH_FORCE_INLINE XXH_TARGET_AVX2 void
-XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc,
- const void* XXH_RESTRICT input,
- const void* XXH_RESTRICT secret)
-{
- XXH_ASSERT((((size_t)acc) & 31) == 0);
- { __m256i* const xacc = (__m256i *) acc;
- /* Unaligned. This is mainly for pointer arithmetic, and because
- * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
- const __m256i* const xinput = (const __m256i *) input;
- /* Unaligned. This is mainly for pointer arithmetic, and because
- * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
- const __m256i* const xsecret = (const __m256i *) secret;
-
- size_t i;
- for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
- /* data_vec = xinput[i]; */
- __m256i const data_vec = _mm256_loadu_si256 (xinput+i);
- /* key_vec = xsecret[i]; */
- __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
- /* data_key = data_vec ^ key_vec; */
- __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
- /* data_key_lo = data_key >> 32; */
- __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
- /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
- __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo);
- /* xacc[i] += swap(data_vec); */
- __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2));
- __m256i const sum = _mm256_add_epi64(xacc[i], data_swap);
- /* xacc[i] += product; */
- xacc[i] = _mm256_add_epi64(product, sum);
- } }
-}
-
-XXH_FORCE_INLINE XXH_TARGET_AVX2 void
-XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
-{
- XXH_ASSERT((((size_t)acc) & 31) == 0);
- { __m256i* const xacc = (__m256i*) acc;
- /* Unaligned. This is mainly for pointer arithmetic, and because
- * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
- const __m256i* const xsecret = (const __m256i *) secret;
- const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1);
-
- size_t i;
- for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
- /* xacc[i] ^= (xacc[i] >> 47) */
- __m256i const acc_vec = xacc[i];
- __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47);
- __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted);
- /* xacc[i] ^= xsecret; */
- __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
- __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
-
- /* xacc[i] *= XXH_PRIME32_1; */
- __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
- __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32);
- __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32);
- xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
- }
- }
-}
-
-XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
-{
- XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0);
- XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6);
- XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64);
- (void)(&XXH_writeLE64);
- XXH_PREFETCH(customSecret);
- { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64);
-
- const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret);
- __m256i* dest = ( __m256i*) customSecret;
-
-# if defined(__GNUC__) || defined(__clang__)
- /*
- * On GCC & Clang, marking 'dest' as modified will cause the compiler:
- * - do not extract the secret from sse registers in the internal loop
- * - use less common registers, and avoid pushing these reg into stack
- */
- XXH_COMPILER_GUARD(dest);
-# endif
- XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */
- XXH_ASSERT(((size_t)dest & 31) == 0);
-
- /* GCC -O2 need unroll loop manually */
- dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed);
- dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed);
- dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed);
- dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed);
- dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed);
- dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed);
- }
-}
-
-#endif
-
-/* x86dispatch always generates SSE2 */
-#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH)
-
-#ifndef XXH_TARGET_SSE2
-# define XXH_TARGET_SSE2 /* disable attribute target */
-#endif
-
-XXH_FORCE_INLINE XXH_TARGET_SSE2 void
-XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc,
- const void* XXH_RESTRICT input,
- const void* XXH_RESTRICT secret)
-{
- /* SSE2 is just a half-scale version of the AVX2 version. */
- XXH_ASSERT((((size_t)acc) & 15) == 0);
- { __m128i* const xacc = (__m128i *) acc;
- /* Unaligned. This is mainly for pointer arithmetic, and because
- * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
- const __m128i* const xinput = (const __m128i *) input;
- /* Unaligned. This is mainly for pointer arithmetic, and because
- * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
- const __m128i* const xsecret = (const __m128i *) secret;
-
- size_t i;
- for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
- /* data_vec = xinput[i]; */
- __m128i const data_vec = _mm_loadu_si128 (xinput+i);
- /* key_vec = xsecret[i]; */
- __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
- /* data_key = data_vec ^ key_vec; */
- __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
- /* data_key_lo = data_key >> 32; */
- __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
- /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
- __m128i const product = _mm_mul_epu32 (data_key, data_key_lo);
- /* xacc[i] += swap(data_vec); */
- __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
- __m128i const sum = _mm_add_epi64(xacc[i], data_swap);
- /* xacc[i] += product; */
- xacc[i] = _mm_add_epi64(product, sum);
- } }
-}
-
-XXH_FORCE_INLINE XXH_TARGET_SSE2 void
-XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
-{
- XXH_ASSERT((((size_t)acc) & 15) == 0);
- { __m128i* const xacc = (__m128i*) acc;
- /* Unaligned. This is mainly for pointer arithmetic, and because
- * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
- const __m128i* const xsecret = (const __m128i *) secret;
- const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1);
-
- size_t i;
- for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
- /* xacc[i] ^= (xacc[i] >> 47) */
- __m128i const acc_vec = xacc[i];
- __m128i const shifted = _mm_srli_epi64 (acc_vec, 47);
- __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted);
- /* xacc[i] ^= xsecret[i]; */
- __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
- __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
-
- /* xacc[i] *= XXH_PRIME32_1; */
- __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
- __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32);
- __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32);
- xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
- }
- }
-}
-
-XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
-{
- XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
- (void)(&XXH_writeLE64);
- { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i);
-
-# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900
- /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */
- XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) };
- __m128i const seed = _mm_load_si128((__m128i const*)seed64x2);
-# else
- __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64);
-# endif
- int i;
-
- const void* const src16 = XXH3_kSecret;
- __m128i* dst16 = (__m128i*) customSecret;
-# if defined(__GNUC__) || defined(__clang__)
- /*
- * On GCC & Clang, marking 'dest' as modified will cause the compiler:
- * - do not extract the secret from sse registers in the internal loop
- * - use less common registers, and avoid pushing these reg into stack
- */
- XXH_COMPILER_GUARD(dst16);
-# endif
- XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */
- XXH_ASSERT(((size_t)dst16 & 15) == 0);
-
- for (i=0; i < nbRounds; ++i) {
- dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed);
- } }
-}
-
-#endif
-
-#if (XXH_VECTOR == XXH_NEON)
-
-/* forward declarations for the scalar routines */
-XXH_FORCE_INLINE void
-XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input,
- void const* XXH_RESTRICT secret, size_t lane);
-
-XXH_FORCE_INLINE void
-XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
- void const* XXH_RESTRICT secret, size_t lane);
-
-/*!
- * @internal
- * @brief The bulk processing loop for NEON.
- *
- * The NEON code path is actually partially scalar when running on AArch64. This
- * is to optimize the pipelining and can have up to 15% speedup depending on the
- * CPU, and it also mitigates some GCC codegen issues.
- *
- * @see XXH3_NEON_LANES for configuring this and details about this optimization.
- */
-XXH_FORCE_INLINE void
-XXH3_accumulate_512_neon( void* XXH_RESTRICT acc,
- const void* XXH_RESTRICT input,
- const void* XXH_RESTRICT secret)
-{
- XXH_ASSERT((((size_t)acc) & 15) == 0);
- XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0);
- {
- uint64x2_t* const xacc = (uint64x2_t *) acc;
- /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
- uint8_t const* const xinput = (const uint8_t *) input;
- uint8_t const* const xsecret = (const uint8_t *) secret;
-
- size_t i;
- /* NEON for the first few lanes (these loops are normally interleaved) */
- for (i=0; i < XXH3_NEON_LANES / 2; i++) {
- /* data_vec = xinput[i]; */
- uint8x16_t data_vec = vld1q_u8(xinput + (i * 16));
- /* key_vec = xsecret[i]; */
- uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16));
- uint64x2_t data_key;
- uint32x2_t data_key_lo, data_key_hi;
- /* xacc[i] += swap(data_vec); */
- uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec);
- uint64x2_t const swapped = vextq_u64(data64, data64, 1);
- xacc[i] = vaddq_u64 (xacc[i], swapped);
- /* data_key = data_vec ^ key_vec; */
- data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec));
- /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF);
- * data_key_hi = (uint32x2_t) (data_key >> 32);
- * data_key = UNDEFINED; */
- XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi);
- /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */
- xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi);
-
- }
- /* Scalar for the remainder. This may be a zero iteration loop. */
- for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
- XXH3_scalarRound(acc, input, secret, i);
- }
- }
-}
-
-XXH_FORCE_INLINE void
-XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
-{
- XXH_ASSERT((((size_t)acc) & 15) == 0);
-
- { uint64x2_t* xacc = (uint64x2_t*) acc;
- uint8_t const* xsecret = (uint8_t const*) secret;
- uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1);
-
- size_t i;
- /* NEON for the first few lanes (these loops are normally interleaved) */
- for (i=0; i < XXH3_NEON_LANES / 2; i++) {
- /* xacc[i] ^= (xacc[i] >> 47); */
- uint64x2_t acc_vec = xacc[i];
- uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47);
- uint64x2_t data_vec = veorq_u64 (acc_vec, shifted);
-
- /* xacc[i] ^= xsecret[i]; */
- uint8x16_t key_vec = vld1q_u8 (xsecret + (i * 16));
- uint64x2_t data_key = veorq_u64 (data_vec, vreinterpretq_u64_u8(key_vec));
-
- /* xacc[i] *= XXH_PRIME32_1 */
- uint32x2_t data_key_lo, data_key_hi;
- /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF);
- * data_key_hi = (uint32x2_t) (xacc[i] >> 32);
- * xacc[i] = UNDEFINED; */
- XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi);
- { /*
- * prod_hi = (data_key >> 32) * XXH_PRIME32_1;
- *
- * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will
- * incorrectly "optimize" this:
- * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b));
- * shifted = vshll_n_u32(tmp, 32);
- * to this:
- * tmp = "vmulq_u64"(a, b); // no such thing!
- * shifted = vshlq_n_u64(tmp, 32);
- *
- * However, unlike SSE, Clang lacks a 64-bit multiply routine
- * for NEON, and it scalarizes two 64-bit multiplies instead.
- *
- * vmull_u32 has the same timing as vmul_u32, and it avoids
- * this bug completely.
- * See https://bugs.llvm.org/show_bug.cgi?id=39967
- */
- uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime);
- /* xacc[i] = prod_hi << 32; */
- xacc[i] = vshlq_n_u64(prod_hi, 32);
- /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */
- xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime);
- }
- }
- /* Scalar for the remainder. This may be a zero iteration loop. */
- for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
- XXH3_scalarScrambleRound(acc, secret, i);
- }
- }
-}
-
-#endif
-
-#if (XXH_VECTOR == XXH_VSX)
-
-XXH_FORCE_INLINE void
-XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc,
- const void* XXH_RESTRICT input,
- const void* XXH_RESTRICT secret)
-{
- /* presumed aligned */
- unsigned int* const xacc = (unsigned int*) acc;
- xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */
- xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */
- xxh_u64x2 const v32 = { 32, 32 };
- size_t i;
- for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
- /* data_vec = xinput[i]; */
- xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i);
- /* key_vec = xsecret[i]; */
- xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i);
- xxh_u64x2 const data_key = data_vec ^ key_vec;
- /* shuffled = (data_key << 32) | (data_key >> 32); */
- xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32);
- /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */
- xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled);
- /* acc_vec = xacc[i]; */
- xxh_u64x2 acc_vec = (xxh_u64x2)vec_xl(0, xacc + 4 * i);
- acc_vec += product;
-
- /* swap high and low halves */
-#ifdef __s390x__
- acc_vec += vec_permi(data_vec, data_vec, 2);
-#else
- acc_vec += vec_xxpermdi(data_vec, data_vec, 2);
-#endif
- /* xacc[i] = acc_vec; */
- vec_xst((xxh_u32x4)acc_vec, 0, xacc + 4 * i);
- }
-}
-
-XXH_FORCE_INLINE void
-XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
-{
- XXH_ASSERT((((size_t)acc) & 15) == 0);
-
- { xxh_u64x2* const xacc = (xxh_u64x2*) acc;
- const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret;
- /* constants */
- xxh_u64x2 const v32 = { 32, 32 };
- xxh_u64x2 const v47 = { 47, 47 };
- xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 };
- size_t i;
- for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
- /* xacc[i] ^= (xacc[i] >> 47); */
- xxh_u64x2 const acc_vec = xacc[i];
- xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47);
-
- /* xacc[i] ^= xsecret[i]; */
- xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i);
- xxh_u64x2 const data_key = data_vec ^ key_vec;
-
- /* xacc[i] *= XXH_PRIME32_1 */
- /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */
- xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime);
- /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */
- xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime);
- xacc[i] = prod_odd + (prod_even << v32);
- } }
-}
-
-#endif
-
-/* scalar variants - universal */
-
-/*!
- * @internal
- * @brief Scalar round for @ref XXH3_accumulate_512_scalar().
- *
- * This is extracted to its own function because the NEON path uses a combination
- * of NEON and scalar.
- */
-XXH_FORCE_INLINE void
-XXH3_scalarRound(void* XXH_RESTRICT acc,
- void const* XXH_RESTRICT input,
- void const* XXH_RESTRICT secret,
- size_t lane)
-{
- xxh_u64* xacc = (xxh_u64*) acc;
- xxh_u8 const* xinput = (xxh_u8 const*) input;
- xxh_u8 const* xsecret = (xxh_u8 const*) secret;
- XXH_ASSERT(lane < XXH_ACC_NB);
- XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
- {
- xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8);
- xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8);
- xacc[lane ^ 1] += data_val; /* swap adjacent lanes */
- xacc[lane] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32);
- }
-}
-
-/*!
- * @internal
- * @brief Processes a 64 byte block of data using the scalar path.
- */
-XXH_FORCE_INLINE void
-XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc,
- const void* XXH_RESTRICT input,
- const void* XXH_RESTRICT secret)
-{
- size_t i;
- for (i=0; i < XXH_ACC_NB; i++) {
- XXH3_scalarRound(acc, input, secret, i);
- }
-}
-
-/*!
- * @internal
- * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar().
- *
- * This is extracted to its own function because the NEON path uses a combination
- * of NEON and scalar.
- */
-XXH_FORCE_INLINE void
-XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
- void const* XXH_RESTRICT secret,
- size_t lane)
-{
- xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */
- const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */
- XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
- XXH_ASSERT(lane < XXH_ACC_NB);
- {
- xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8);
- xxh_u64 acc64 = xacc[lane];
- acc64 = XXH_xorshift64(acc64, 47);
- acc64 ^= key64;
- acc64 *= XXH_PRIME32_1;
- xacc[lane] = acc64;
- }
-}
-
-/*!
- * @internal
- * @brief Scrambles the accumulators after a large chunk has been read
- */
-XXH_FORCE_INLINE void
-XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
-{
- size_t i;
- for (i=0; i < XXH_ACC_NB; i++) {
- XXH3_scalarScrambleRound(acc, secret, i);
- }
-}
-
-XXH_FORCE_INLINE void
-XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
-{
- /*
- * We need a separate pointer for the hack below,
- * which requires a non-const pointer.
- * Any decent compiler will optimize this out otherwise.
- */
- const xxh_u8* kSecretPtr = XXH3_kSecret;
- XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
-
-#if defined(__clang__) && defined(__aarch64__)
- /*
- * UGLY HACK:
- * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are
- * placed sequentially, in order, at the top of the unrolled loop.
- *
- * While MOVK is great for generating constants (2 cycles for a 64-bit
- * constant compared to 4 cycles for LDR), it fights for bandwidth with
- * the arithmetic instructions.
- *
- * I L S
- * MOVK
- * MOVK
- * MOVK
- * MOVK
- * ADD
- * SUB STR
- * STR
- * By forcing loads from memory (as the asm line causes Clang to assume
- * that XXH3_kSecretPtr has been changed), the pipelines are used more
- * efficiently:
- * I L S
- * LDR
- * ADD LDR
- * SUB STR
- * STR
- *
- * See XXH3_NEON_LANES for details on the pipsline.
- *
- * XXH3_64bits_withSeed, len == 256, Snapdragon 835
- * without hack: 2654.4 MB/s
- * with hack: 3202.9 MB/s
- */
- XXH_COMPILER_GUARD(kSecretPtr);
-#endif
- /*
- * Note: in debug mode, this overrides the asm optimization
- * and Clang will emit MOVK chains again.
- */
- XXH_ASSERT(kSecretPtr == XXH3_kSecret);
-
- { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
- int i;
- for (i=0; i < nbRounds; i++) {
- /*
- * The asm hack causes Clang to assume that kSecretPtr aliases with
- * customSecret, and on aarch64, this prevented LDP from merging two
- * loads together for free. Putting the loads together before the stores
- * properly generates LDP.
- */
- xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64;
- xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64;
- XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo);
- XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi);
- } }
-}
-
-
-typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*);
-typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*);
-typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64);
-
-
-#if (XXH_VECTOR == XXH_AVX512)
-
-#define XXH3_accumulate_512 XXH3_accumulate_512_avx512
-#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512
-#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512
-
-#elif (XXH_VECTOR == XXH_AVX2)
-
-#define XXH3_accumulate_512 XXH3_accumulate_512_avx2
-#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2
-#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2
-
-#elif (XXH_VECTOR == XXH_SSE2)
-
-#define XXH3_accumulate_512 XXH3_accumulate_512_sse2
-#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2
-#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2
-
-#elif (XXH_VECTOR == XXH_NEON)
-
-#define XXH3_accumulate_512 XXH3_accumulate_512_neon
-#define XXH3_scrambleAcc XXH3_scrambleAcc_neon
-#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
-
-#elif (XXH_VECTOR == XXH_VSX)
-
-#define XXH3_accumulate_512 XXH3_accumulate_512_vsx
-#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx
-#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
-
-#else /* scalar */
-
-#define XXH3_accumulate_512 XXH3_accumulate_512_scalar
-#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar
-#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
-
-#endif
-
-
-
-#ifndef XXH_PREFETCH_DIST
-# ifdef __clang__
-# define XXH_PREFETCH_DIST 320
-# else
-# if (XXH_VECTOR == XXH_AVX512)
-# define XXH_PREFETCH_DIST 512
-# else
-# define XXH_PREFETCH_DIST 384
-# endif
-# endif /* __clang__ */
-#endif /* XXH_PREFETCH_DIST */
-
-/*
- * XXH3_accumulate()
- * Loops over XXH3_accumulate_512().
- * Assumption: nbStripes will not overflow the secret size
- */
-XXH_FORCE_INLINE void
-XXH3_accumulate( xxh_u64* XXH_RESTRICT acc,
- const xxh_u8* XXH_RESTRICT input,
- const xxh_u8* XXH_RESTRICT secret,
- size_t nbStripes,
- XXH3_f_accumulate_512 f_acc512)
-{
- size_t n;
- for (n = 0; n < nbStripes; n++ ) {
- const xxh_u8* const in = input + n*XXH_STRIPE_LEN;
- XXH_PREFETCH(in + XXH_PREFETCH_DIST);
- f_acc512(acc,
- in,
- secret + n*XXH_SECRET_CONSUME_RATE);
- }
-}
-
-XXH_FORCE_INLINE void
-XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc,
- const xxh_u8* XXH_RESTRICT input, size_t len,
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
- XXH3_f_accumulate_512 f_acc512,
- XXH3_f_scrambleAcc f_scramble)
-{
- size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
- size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock;
- size_t const nb_blocks = (len - 1) / block_len;
-
- size_t n;
-
- XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
-
- for (n = 0; n < nb_blocks; n++) {
- XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512);
- f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN);
- }
-
- /* last partial block */
- XXH_ASSERT(len > XXH_STRIPE_LEN);
- { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
- XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
- XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512);
-
- /* last stripe */
- { const xxh_u8* const p = input + len - XXH_STRIPE_LEN;
-#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */
- f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START);
- } }
-}
-
-XXH_FORCE_INLINE xxh_u64
-XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret)
-{
- return XXH3_mul128_fold64(
- acc[0] ^ XXH_readLE64(secret),
- acc[1] ^ XXH_readLE64(secret+8) );
-}
-
-static XXH64_hash_t
-XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
-{
- xxh_u64 result64 = start;
- size_t i = 0;
-
- for (i = 0; i < 4; i++) {
- result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i);
-#if defined(__clang__) /* Clang */ \
- && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \
- && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
- && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
- /*
- * UGLY HACK:
- * Prevent autovectorization on Clang ARMv7-a. Exact same problem as
- * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b.
- * XXH3_64bits, len == 256, Snapdragon 835:
- * without hack: 2063.7 MB/s
- * with hack: 2560.7 MB/s
- */
- XXH_COMPILER_GUARD(result64);
-#endif
- }
-
- return XXH3_avalanche(result64);
-}
-
-#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
- XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
-
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len,
- const void* XXH_RESTRICT secret, size_t secretSize,
- XXH3_f_accumulate_512 f_acc512,
- XXH3_f_scrambleAcc f_scramble)
-{
- XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
-
- XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble);
-
- /* converge into final hash */
- XXH_STATIC_ASSERT(sizeof(acc) == 64);
- /* do not align on 8, so that the secret is different from the accumulator */
-#define XXH_SECRET_MERGEACCS_START 11
- XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
- return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1);
-}
-
-/*
- * It's important for performance to transmit secret's size (when it's static)
- * so that the compiler can properly optimize the vectorized loop.
- * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set.
- */
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len,
- XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
-{
- (void)seed64;
- return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc);
-}
-
-/*
- * It's preferable for performance that XXH3_hashLong is not inlined,
- * as it results in a smaller function for small data, easier to the instruction cache.
- * Note that inside this no_inline function, we do inline the internal loop,
- * and provide a statically defined secret size to allow optimization of vector loop.
- */
-XXH_NO_INLINE XXH64_hash_t
-XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len,
- XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
-{
- (void)seed64; (void)secret; (void)secretLen;
- return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc);
-}
-
-/*
- * XXH3_hashLong_64b_withSeed():
- * Generate a custom key based on alteration of default XXH3_kSecret with the seed,
- * and then use this key for long mode hashing.
- *
- * This operation is decently fast but nonetheless costs a little bit of time.
- * Try to avoid it whenever possible (typically when seed==0).
- *
- * It's important for performance that XXH3_hashLong is not inlined. Not sure
- * why (uop cache maybe?), but the difference is large and easily measurable.
- */
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len,
- XXH64_hash_t seed,
- XXH3_f_accumulate_512 f_acc512,
- XXH3_f_scrambleAcc f_scramble,
- XXH3_f_initCustomSecret f_initSec)
-{
- if (seed == 0)
- return XXH3_hashLong_64b_internal(input, len,
- XXH3_kSecret, sizeof(XXH3_kSecret),
- f_acc512, f_scramble);
- { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
- f_initSec(secret, seed);
- return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret),
- f_acc512, f_scramble);
- }
-}
-
-/*
- * It's important for performance that XXH3_hashLong is not inlined.
- */
-XXH_NO_INLINE XXH64_hash_t
-XXH3_hashLong_64b_withSeed(const void* input, size_t len,
- XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen)
-{
- (void)secret; (void)secretLen;
- return XXH3_hashLong_64b_withSeed_internal(input, len, seed,
- XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret);
-}
-
-
-typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t,
- XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t);
-
-XXH_FORCE_INLINE XXH64_hash_t
-XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len,
- XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
- XXH3_hashLong64_f f_hashLong)
-{
- XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
- /*
- * If an action is to be taken if `secretLen` condition is not respected,
- * it should be done here.
- * For now, it's a contract pre-condition.
- * Adding a check and a branch here would cost performance at every hash.
- * Also, note that function signature doesn't offer room to return an error.
- */
- if (len <= 16)
- return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
- if (len <= 128)
- return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
- if (len <= XXH3_MIDSIZE_MAX)
- return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
- return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen);
-}
-
-
-/* === Public entry point === */
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len)
-{
- return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default);
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH64_hash_t
-XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
-{
- return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret);
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH64_hash_t
-XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
-{
- return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed);
-}
-
-XXH_PUBLIC_API XXH64_hash_t
-XXH3_64bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed)
-{
- if (len <= XXH3_MIDSIZE_MAX)
- return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
- return XXH3_hashLong_64b_withSecret(input, len, seed, (const xxh_u8*)secret, secretSize);
-}
-
-
-/* === XXH3 streaming === */
-
-/*
- * Malloc's a pointer that is always aligned to align.
- *
- * This must be freed with `XXH_alignedFree()`.
- *
- * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte
- * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2
- * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON.
- *
- * This underalignment previously caused a rather obvious crash which went
- * completely unnoticed due to XXH3_createState() not actually being tested.
- * Credit to RedSpah for noticing this bug.
- *
- * The alignment is done manually: Functions like posix_memalign or _mm_malloc
- * are avoided: To maintain portability, we would have to write a fallback
- * like this anyways, and besides, testing for the existence of library
- * functions without relying on external build tools is impossible.
- *
- * The method is simple: Overallocate, manually align, and store the offset
- * to the original behind the returned pointer.
- *
- * Align must be a power of 2 and 8 <= align <= 128.
- */
-static void* XXH_alignedMalloc(size_t s, size_t align)
-{
- XXH_ASSERT(align <= 128 && align >= 8); /* range check */
- XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */
- XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */
- { /* Overallocate to make room for manual realignment and an offset byte */
- xxh_u8* base = (xxh_u8*)XXH_malloc(s + align);
- if (base != NULL) {
- /*
- * Get the offset needed to align this pointer.
- *
- * Even if the returned pointer is aligned, there will always be
- * at least one byte to store the offset to the original pointer.
- */
- size_t offset = align - ((size_t)base & (align - 1)); /* base % align */
- /* Add the offset for the now-aligned pointer */
- xxh_u8* ptr = base + offset;
-
- XXH_ASSERT((size_t)ptr % align == 0);
-
- /* Store the offset immediately before the returned pointer. */
- ptr[-1] = (xxh_u8)offset;
- return ptr;
- }
- return NULL;
- }
-}
-/*
- * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass
- * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout.
- */
-static void XXH_alignedFree(void* p)
-{
- if (p != NULL) {
- xxh_u8* ptr = (xxh_u8*)p;
- /* Get the offset byte we added in XXH_malloc. */
- xxh_u8 offset = ptr[-1];
- /* Free the original malloc'd pointer */
- xxh_u8* base = ptr - offset;
- XXH_free(base);
- }
-}
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
-{
- XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64);
- if (state==NULL) return NULL;
- XXH3_INITSTATE(state);
- return state;
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
-{
- XXH_alignedFree(statePtr);
- return XXH_OK;
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API void
-XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state)
-{
- XXH_memcpy(dst_state, src_state, sizeof(*dst_state));
-}
-
-static void
-XXH3_reset_internal(XXH3_state_t* statePtr,
- XXH64_hash_t seed,
- const void* secret, size_t secretSize)
-{
- size_t const initStart = offsetof(XXH3_state_t, bufferedSize);
- size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart;
- XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart);
- XXH_ASSERT(statePtr != NULL);
- /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */
- memset((char*)statePtr + initStart, 0, initLength);
- statePtr->acc[0] = XXH_PRIME32_3;
- statePtr->acc[1] = XXH_PRIME64_1;
- statePtr->acc[2] = XXH_PRIME64_2;
- statePtr->acc[3] = XXH_PRIME64_3;
- statePtr->acc[4] = XXH_PRIME64_4;
- statePtr->acc[5] = XXH_PRIME32_2;
- statePtr->acc[6] = XXH_PRIME64_5;
- statePtr->acc[7] = XXH_PRIME32_1;
- statePtr->seed = seed;
- statePtr->useSeed = (seed != 0);
- statePtr->extSecret = (const unsigned char*)secret;
- XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
- statePtr->secretLimit = secretSize - XXH_STRIPE_LEN;
- statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE;
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH3_64bits_reset(XXH3_state_t* statePtr)
-{
- if (statePtr == NULL) return XXH_ERROR;
- XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
- return XXH_OK;
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
-{
- if (statePtr == NULL) return XXH_ERROR;
- XXH3_reset_internal(statePtr, 0, secret, secretSize);
- if (secret == NULL) return XXH_ERROR;
- if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
- return XXH_OK;
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
-{
- if (statePtr == NULL) return XXH_ERROR;
- if (seed==0) return XXH3_64bits_reset(statePtr);
- if ((seed != statePtr->seed) || (statePtr->extSecret != NULL))
- XXH3_initCustomSecret(statePtr->customSecret, seed);
- XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE);
- return XXH_OK;
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed64)
-{
- if (statePtr == NULL) return XXH_ERROR;
- if (secret == NULL) return XXH_ERROR;
- if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
- XXH3_reset_internal(statePtr, seed64, secret, secretSize);
- statePtr->useSeed = 1; /* always, even if seed64==0 */
- return XXH_OK;
-}
-
-/* Note : when XXH3_consumeStripes() is invoked,
- * there must be a guarantee that at least one more byte must be consumed from input
- * so that the function can blindly consume all stripes using the "normal" secret segment */
-XXH_FORCE_INLINE void
-XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc,
- size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock,
- const xxh_u8* XXH_RESTRICT input, size_t nbStripes,
- const xxh_u8* XXH_RESTRICT secret, size_t secretLimit,
- XXH3_f_accumulate_512 f_acc512,
- XXH3_f_scrambleAcc f_scramble)
-{
- XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */
- XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock);
- if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) {
- /* need a scrambling operation */
- size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr;
- size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock;
- XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512);
- f_scramble(acc, secret + secretLimit);
- XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512);
- *nbStripesSoFarPtr = nbStripesAfterBlock;
- } else {
- XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512);
- *nbStripesSoFarPtr += nbStripes;
- }
-}
-
-#ifndef XXH3_STREAM_USE_STACK
-# ifndef __clang__ /* clang doesn't need additional stack space */
-# define XXH3_STREAM_USE_STACK 1
-# endif
-#endif
-/*
- * Both XXH3_64bits_update and XXH3_128bits_update use this routine.
- */
-XXH_FORCE_INLINE XXH_errorcode
-XXH3_update(XXH3_state_t* XXH_RESTRICT const state,
- const xxh_u8* XXH_RESTRICT input, size_t len,
- XXH3_f_accumulate_512 f_acc512,
- XXH3_f_scrambleAcc f_scramble)
-{
- if (input==NULL) {
- XXH_ASSERT(len == 0);
- return XXH_OK;
- }
-
- XXH_ASSERT(state != NULL);
- { const xxh_u8* const bEnd = input + len;
- const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
-#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
- /* For some reason, gcc and MSVC seem to suffer greatly
- * when operating accumulators directly into state.
- * Operating into stack space seems to enable proper optimization.
- * clang, on the other hand, doesn't seem to need this trick */
- XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; memcpy(acc, state->acc, sizeof(acc));
-#else
- xxh_u64* XXH_RESTRICT const acc = state->acc;
-#endif
- state->totalLen += len;
- XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE);
-
- /* small input : just fill in tmp buffer */
- if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) {
- XXH_memcpy(state->buffer + state->bufferedSize, input, len);
- state->bufferedSize += (XXH32_hash_t)len;
- return XXH_OK;
- }
-
- /* total input is now > XXH3_INTERNALBUFFER_SIZE */
- #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN)
- XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */
-
- /*
- * Internal buffer is partially filled (always, except at beginning)
- * Complete it, then consume it.
- */
- if (state->bufferedSize) {
- size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize;
- XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize);
- input += loadSize;
- XXH3_consumeStripes(acc,
- &state->nbStripesSoFar, state->nbStripesPerBlock,
- state->buffer, XXH3_INTERNALBUFFER_STRIPES,
- secret, state->secretLimit,
- f_acc512, f_scramble);
- state->bufferedSize = 0;
- }
- XXH_ASSERT(input < bEnd);
-
- /* large input to consume : ingest per full block */
- if ((size_t)(bEnd - input) > state->nbStripesPerBlock * XXH_STRIPE_LEN) {
- size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN;
- XXH_ASSERT(state->nbStripesPerBlock >= state->nbStripesSoFar);
- /* join to current block's end */
- { size_t const nbStripesToEnd = state->nbStripesPerBlock - state->nbStripesSoFar;
- XXH_ASSERT(nbStripesToEnd <= nbStripes);
- XXH3_accumulate(acc, input, secret + state->nbStripesSoFar * XXH_SECRET_CONSUME_RATE, nbStripesToEnd, f_acc512);
- f_scramble(acc, secret + state->secretLimit);
- state->nbStripesSoFar = 0;
- input += nbStripesToEnd * XXH_STRIPE_LEN;
- nbStripes -= nbStripesToEnd;
- }
- /* consume per entire blocks */
- while(nbStripes >= state->nbStripesPerBlock) {
- XXH3_accumulate(acc, input, secret, state->nbStripesPerBlock, f_acc512);
- f_scramble(acc, secret + state->secretLimit);
- input += state->nbStripesPerBlock * XXH_STRIPE_LEN;
- nbStripes -= state->nbStripesPerBlock;
- }
- /* consume last partial block */
- XXH3_accumulate(acc, input, secret, nbStripes, f_acc512);
- input += nbStripes * XXH_STRIPE_LEN;
- XXH_ASSERT(input < bEnd); /* at least some bytes left */
- state->nbStripesSoFar = nbStripes;
- /* buffer predecessor of last partial stripe */
- XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
- XXH_ASSERT(bEnd - input <= XXH_STRIPE_LEN);
- } else {
- /* content to consume <= block size */
- /* Consume input by a multiple of internal buffer size */
- if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) {
- const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE;
- do {
- XXH3_consumeStripes(acc,
- &state->nbStripesSoFar, state->nbStripesPerBlock,
- input, XXH3_INTERNALBUFFER_STRIPES,
- secret, state->secretLimit,
- f_acc512, f_scramble);
- input += XXH3_INTERNALBUFFER_SIZE;
- } while (input<limit);
- /* buffer predecessor of last partial stripe */
- XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
- }
- }
-
- /* Some remaining input (always) : buffer it */
- XXH_ASSERT(input < bEnd);
- XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE);
- XXH_ASSERT(state->bufferedSize == 0);
- XXH_memcpy(state->buffer, input, (size_t)(bEnd-input));
- state->bufferedSize = (XXH32_hash_t)(bEnd-input);
-#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
- /* save stack accumulators into state */
- memcpy(state->acc, acc, sizeof(acc));
-#endif
- }
-
- return XXH_OK;
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len)
-{
- return XXH3_update(state, (const xxh_u8*)input, len,
- XXH3_accumulate_512, XXH3_scrambleAcc);
-}
-
-
-XXH_FORCE_INLINE void
-XXH3_digest_long (XXH64_hash_t* acc,
- const XXH3_state_t* state,
- const unsigned char* secret)
-{
- /*
- * Digest on a local copy. This way, the state remains unaltered, and it can
- * continue ingesting more input afterwards.
- */
- XXH_memcpy(acc, state->acc, sizeof(state->acc));
- if (state->bufferedSize >= XXH_STRIPE_LEN) {
- size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN;
- size_t nbStripesSoFar = state->nbStripesSoFar;
- XXH3_consumeStripes(acc,
- &nbStripesSoFar, state->nbStripesPerBlock,
- state->buffer, nbStripes,
- secret, state->secretLimit,
- XXH3_accumulate_512, XXH3_scrambleAcc);
- /* last stripe */
- XXH3_accumulate_512(acc,
- state->buffer + state->bufferedSize - XXH_STRIPE_LEN,
- secret + state->secretLimit - XXH_SECRET_LASTACC_START);
- } else { /* bufferedSize < XXH_STRIPE_LEN */
- xxh_u8 lastStripe[XXH_STRIPE_LEN];
- size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize;
- XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */
- XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize);
- XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize);
- XXH3_accumulate_512(acc,
- lastStripe,
- secret + state->secretLimit - XXH_SECRET_LASTACC_START);
- }
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state)
-{
- const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
- if (state->totalLen > XXH3_MIDSIZE_MAX) {
- XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
- XXH3_digest_long(acc, state, secret);
- return XXH3_mergeAccs(acc,
- secret + XXH_SECRET_MERGEACCS_START,
- (xxh_u64)state->totalLen * XXH_PRIME64_1);
- }
- /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
- if (state->useSeed)
- return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
- return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen),
- secret, state->secretLimit + XXH_STRIPE_LEN);
-}
-
-
-
-/* ==========================================
- * XXH3 128 bits (a.k.a XXH128)
- * ==========================================
- * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
- * even without counting the significantly larger output size.
- *
- * For example, extra steps are taken to avoid the seed-dependent collisions
- * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B).
- *
- * This strength naturally comes at the cost of some speed, especially on short
- * lengths. Note that longer hashes are about as fast as the 64-bit version
- * due to it using only a slight modification of the 64-bit loop.
- *
- * XXH128 is also more oriented towards 64-bit machines. It is still extremely
- * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64).
- */
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
-{
- /* A doubled version of 1to3_64b with different constants. */
- XXH_ASSERT(input != NULL);
- XXH_ASSERT(1 <= len && len <= 3);
- XXH_ASSERT(secret != NULL);
- /*
- * len = 1: combinedl = { input[0], 0x01, input[0], input[0] }
- * len = 2: combinedl = { input[1], 0x02, input[0], input[1] }
- * len = 3: combinedl = { input[2], 0x03, input[0], input[1] }
- */
- { xxh_u8 const c1 = input[0];
- xxh_u8 const c2 = input[len >> 1];
- xxh_u8 const c3 = input[len - 1];
- xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24)
- | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
- xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13);
- xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
- xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed;
- xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl;
- xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph;
- XXH128_hash_t h128;
- h128.low64 = XXH64_avalanche(keyed_lo);
- h128.high64 = XXH64_avalanche(keyed_hi);
- return h128;
- }
-}
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
-{
- XXH_ASSERT(input != NULL);
- XXH_ASSERT(secret != NULL);
- XXH_ASSERT(4 <= len && len <= 8);
- seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
- { xxh_u32 const input_lo = XXH_readLE32(input);
- xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
- xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32);
- xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed;
- xxh_u64 const keyed = input_64 ^ bitflip;
-
- /* Shift len to the left to ensure it is even, this avoids even multiplies. */
- XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2));
-
- m128.high64 += (m128.low64 << 1);
- m128.low64 ^= (m128.high64 >> 3);
-
- m128.low64 = XXH_xorshift64(m128.low64, 35);
- m128.low64 *= 0x9FB21C651E98DF25ULL;
- m128.low64 = XXH_xorshift64(m128.low64, 28);
- m128.high64 = XXH3_avalanche(m128.high64);
- return m128;
- }
-}
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
-{
- XXH_ASSERT(input != NULL);
- XXH_ASSERT(secret != NULL);
- XXH_ASSERT(9 <= len && len <= 16);
- { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed;
- xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed;
- xxh_u64 const input_lo = XXH_readLE64(input);
- xxh_u64 input_hi = XXH_readLE64(input + len - 8);
- XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1);
- /*
- * Put len in the middle of m128 to ensure that the length gets mixed to
- * both the low and high bits in the 128x64 multiply below.
- */
- m128.low64 += (xxh_u64)(len - 1) << 54;
- input_hi ^= bitfliph;
- /*
- * Add the high 32 bits of input_hi to the high 32 bits of m128, then
- * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to
- * the high 64 bits of m128.
- *
- * The best approach to this operation is different on 32-bit and 64-bit.
- */
- if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */
- /*
- * 32-bit optimized version, which is more readable.
- *
- * On 32-bit, it removes an ADC and delays a dependency between the two
- * halves of m128.high64, but it generates an extra mask on 64-bit.
- */
- m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2);
- } else {
- /*
- * 64-bit optimized (albeit more confusing) version.
- *
- * Uses some properties of addition and multiplication to remove the mask:
- *
- * Let:
- * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF)
- * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000)
- * c = XXH_PRIME32_2
- *
- * a + (b * c)
- * Inverse Property: x + y - x == y
- * a + (b * (1 + c - 1))
- * Distributive Property: x * (y + z) == (x * y) + (x * z)
- * a + (b * 1) + (b * (c - 1))
- * Identity Property: x * 1 == x
- * a + b + (b * (c - 1))
- *
- * Substitute a, b, and c:
- * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
- *
- * Since input_hi.hi + input_hi.lo == input_hi, we get this:
- * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
- */
- m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1);
- }
- /* m128 ^= XXH_swap64(m128 >> 64); */
- m128.low64 ^= XXH_swap64(m128.high64);
-
- { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */
- XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2);
- h128.high64 += m128.high64 * XXH_PRIME64_2;
-
- h128.low64 = XXH3_avalanche(h128.low64);
- h128.high64 = XXH3_avalanche(h128.high64);
- return h128;
- } }
-}
-
-/*
- * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN
- */
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
-{
- XXH_ASSERT(len <= 16);
- { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed);
- if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed);
- if (len) return XXH3_len_1to3_128b(input, len, secret, seed);
- { XXH128_hash_t h128;
- xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72);
- xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88);
- h128.low64 = XXH64_avalanche(seed ^ bitflipl);
- h128.high64 = XXH64_avalanche( seed ^ bitfliph);
- return h128;
- } }
-}
-
-/*
- * A bit slower than XXH3_mix16B, but handles multiply by zero better.
- */
-XXH_FORCE_INLINE XXH128_hash_t
-XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2,
- const xxh_u8* secret, XXH64_hash_t seed)
-{
- acc.low64 += XXH3_mix16B (input_1, secret+0, seed);
- acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8);
- acc.high64 += XXH3_mix16B (input_2, secret+16, seed);
- acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8);
- return acc;
-}
-
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
- XXH64_hash_t seed)
-{
- XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
- XXH_ASSERT(16 < len && len <= 128);
-
- { XXH128_hash_t acc;
- acc.low64 = len * XXH_PRIME64_1;
- acc.high64 = 0;
- if (len > 32) {
- if (len > 64) {
- if (len > 96) {
- acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed);
- }
- acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed);
- }
- acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed);
- }
- acc = XXH128_mix32B(acc, input, input+len-16, secret, seed);
- { XXH128_hash_t h128;
- h128.low64 = acc.low64 + acc.high64;
- h128.high64 = (acc.low64 * XXH_PRIME64_1)
- + (acc.high64 * XXH_PRIME64_4)
- + ((len - seed) * XXH_PRIME64_2);
- h128.low64 = XXH3_avalanche(h128.low64);
- h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
- return h128;
- }
- }
-}
-
-XXH_NO_INLINE XXH128_hash_t
-XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
- XXH64_hash_t seed)
-{
- XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
- XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
-
- { XXH128_hash_t acc;
- int const nbRounds = (int)len / 32;
- int i;
- acc.low64 = len * XXH_PRIME64_1;
- acc.high64 = 0;
- for (i=0; i<4; i++) {
- acc = XXH128_mix32B(acc,
- input + (32 * i),
- input + (32 * i) + 16,
- secret + (32 * i),
- seed);
- }
- acc.low64 = XXH3_avalanche(acc.low64);
- acc.high64 = XXH3_avalanche(acc.high64);
- XXH_ASSERT(nbRounds >= 4);
- for (i=4 ; i < nbRounds; i++) {
- acc = XXH128_mix32B(acc,
- input + (32 * i),
- input + (32 * i) + 16,
- secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)),
- seed);
- }
- /* last bytes */
- acc = XXH128_mix32B(acc,
- input + len - 16,
- input + len - 32,
- secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16,
- 0ULL - seed);
-
- { XXH128_hash_t h128;
- h128.low64 = acc.low64 + acc.high64;
- h128.high64 = (acc.low64 * XXH_PRIME64_1)
- + (acc.high64 * XXH_PRIME64_4)
- + ((len - seed) * XXH_PRIME64_2);
- h128.low64 = XXH3_avalanche(h128.low64);
- h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
- return h128;
- }
- }
-}
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len,
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
- XXH3_f_accumulate_512 f_acc512,
- XXH3_f_scrambleAcc f_scramble)
-{
- XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
-
- XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble);
-
- /* converge into final hash */
- XXH_STATIC_ASSERT(sizeof(acc) == 64);
- XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
- { XXH128_hash_t h128;
- h128.low64 = XXH3_mergeAccs(acc,
- secret + XXH_SECRET_MERGEACCS_START,
- (xxh_u64)len * XXH_PRIME64_1);
- h128.high64 = XXH3_mergeAccs(acc,
- secret + secretSize
- - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
- ~((xxh_u64)len * XXH_PRIME64_2));
- return h128;
- }
-}
-
-/*
- * It's important for performance that XXH3_hashLong is not inlined.
- */
-XXH_NO_INLINE XXH128_hash_t
-XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len,
- XXH64_hash_t seed64,
- const void* XXH_RESTRICT secret, size_t secretLen)
-{
- (void)seed64; (void)secret; (void)secretLen;
- return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret),
- XXH3_accumulate_512, XXH3_scrambleAcc);
-}
-
-/*
- * It's important for performance to pass @secretLen (when it's static)
- * to the compiler, so that it can properly optimize the vectorized loop.
- */
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len,
- XXH64_hash_t seed64,
- const void* XXH_RESTRICT secret, size_t secretLen)
-{
- (void)seed64;
- return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen,
- XXH3_accumulate_512, XXH3_scrambleAcc);
-}
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len,
- XXH64_hash_t seed64,
- XXH3_f_accumulate_512 f_acc512,
- XXH3_f_scrambleAcc f_scramble,
- XXH3_f_initCustomSecret f_initSec)
-{
- if (seed64 == 0)
- return XXH3_hashLong_128b_internal(input, len,
- XXH3_kSecret, sizeof(XXH3_kSecret),
- f_acc512, f_scramble);
- { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
- f_initSec(secret, seed64);
- return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret),
- f_acc512, f_scramble);
- }
-}
-
-/*
- * It's important for performance that XXH3_hashLong is not inlined.
- */
-XXH_NO_INLINE XXH128_hash_t
-XXH3_hashLong_128b_withSeed(const void* input, size_t len,
- XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen)
-{
- (void)secret; (void)secretLen;
- return XXH3_hashLong_128b_withSeed_internal(input, len, seed64,
- XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret);
-}
-
-typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t,
- XXH64_hash_t, const void* XXH_RESTRICT, size_t);
-
-XXH_FORCE_INLINE XXH128_hash_t
-XXH3_128bits_internal(const void* input, size_t len,
- XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
- XXH3_hashLong128_f f_hl128)
-{
- XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
- /*
- * If an action is to be taken if `secret` conditions are not respected,
- * it should be done here.
- * For now, it's a contract pre-condition.
- * Adding a check and a branch here would cost performance at every hash.
- */
- if (len <= 16)
- return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
- if (len <= 128)
- return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
- if (len <= XXH3_MIDSIZE_MAX)
- return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
- return f_hl128(input, len, seed64, secret, secretLen);
-}
-
-
-/* === Public XXH128 API === */
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len)
-{
- return XXH3_128bits_internal(input, len, 0,
- XXH3_kSecret, sizeof(XXH3_kSecret),
- XXH3_hashLong_128b_default);
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH128_hash_t
-XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
-{
- return XXH3_128bits_internal(input, len, 0,
- (const xxh_u8*)secret, secretSize,
- XXH3_hashLong_128b_withSecret);
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH128_hash_t
-XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
-{
- return XXH3_128bits_internal(input, len, seed,
- XXH3_kSecret, sizeof(XXH3_kSecret),
- XXH3_hashLong_128b_withSeed);
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH128_hash_t
-XXH3_128bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed)
-{
- if (len <= XXH3_MIDSIZE_MAX)
- return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
- return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize);
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH128_hash_t
-XXH128(const void* input, size_t len, XXH64_hash_t seed)
-{
- return XXH3_128bits_withSeed(input, len, seed);
-}
-
-
-/* === XXH3 128-bit streaming === */
-
-/*
- * All initialization and update functions are identical to 64-bit streaming variant.
- * The only difference is the finalization routine.
- */
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH3_128bits_reset(XXH3_state_t* statePtr)
-{
- return XXH3_64bits_reset(statePtr);
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
-{
- return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize);
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
-{
- return XXH3_64bits_reset_withSeed(statePtr, seed);
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed)
-{
- return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed);
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len)
-{
- return XXH3_update(state, (const xxh_u8*)input, len,
- XXH3_accumulate_512, XXH3_scrambleAcc);
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state)
-{
- const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
- if (state->totalLen > XXH3_MIDSIZE_MAX) {
- XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
- XXH3_digest_long(acc, state, secret);
- XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
- { XXH128_hash_t h128;
- h128.low64 = XXH3_mergeAccs(acc,
- secret + XXH_SECRET_MERGEACCS_START,
- (xxh_u64)state->totalLen * XXH_PRIME64_1);
- h128.high64 = XXH3_mergeAccs(acc,
- secret + state->secretLimit + XXH_STRIPE_LEN
- - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
- ~((xxh_u64)state->totalLen * XXH_PRIME64_2));
- return h128;
- }
- }
- /* len <= XXH3_MIDSIZE_MAX : short code */
- if (state->seed)
- return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
- return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen),
- secret, state->secretLimit + XXH_STRIPE_LEN);
-}
-
-/* 128-bit utility functions */
-
-#include <string.h> /* memcmp, memcpy */
-
-/* return : 1 is equal, 0 if different */
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
-{
- /* note : XXH128_hash_t is compact, it has no padding byte */
- return !(memcmp(&h1, &h2, sizeof(h1)));
-}
-
-/* This prototype is compatible with stdlib's qsort().
- * return : >0 if *h128_1 > *h128_2
- * <0 if *h128_1 < *h128_2
- * =0 if *h128_1 == *h128_2 */
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2)
-{
- XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1;
- XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2;
- int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
- /* note : bets that, in most cases, hash values are different */
- if (hcmp) return hcmp;
- return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
-}
-
-
-/*====== Canonical representation ======*/
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API void
-XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash)
-{
- XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
- if (XXH_CPU_LITTLE_ENDIAN) {
- hash.high64 = XXH_swap64(hash.high64);
- hash.low64 = XXH_swap64(hash.low64);
- }
- XXH_memcpy(dst, &hash.high64, sizeof(hash.high64));
- XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH128_hash_t
-XXH128_hashFromCanonical(const XXH128_canonical_t* src)
-{
- XXH128_hash_t h;
- h.high64 = XXH_readBE64(src);
- h.low64 = XXH_readBE64(src->digest + 8);
- return h;
-}
-
-
-
-/* ==========================================
- * Secret generators
- * ==========================================
- */
-#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x))
-
-XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128)
-{
- XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 );
- XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 );
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API XXH_errorcode
-XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize)
-{
-#if (XXH_DEBUGLEVEL >= 1)
- XXH_ASSERT(secretBuffer != NULL);
- XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
-#else
- /* production mode, assert() are disabled */
- if (secretBuffer == NULL) return XXH_ERROR;
- if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
-#endif
-
- if (customSeedSize == 0) {
- customSeed = XXH3_kSecret;
- customSeedSize = XXH_SECRET_DEFAULT_SIZE;
- }
-#if (XXH_DEBUGLEVEL >= 1)
- XXH_ASSERT(customSeed != NULL);
-#else
- if (customSeed == NULL) return XXH_ERROR;
-#endif
-
- /* Fill secretBuffer with a copy of customSeed - repeat as needed */
- { size_t pos = 0;
- while (pos < secretSize) {
- size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize);
- memcpy((char*)secretBuffer + pos, customSeed, toCopy);
- pos += toCopy;
- } }
-
- { size_t const nbSeg16 = secretSize / 16;
- size_t n;
- XXH128_canonical_t scrambler;
- XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0));
- for (n=0; n<nbSeg16; n++) {
- XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n);
- XXH3_combine16((char*)secretBuffer + n*16, h128);
- }
- /* last segment */
- XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler));
- }
- return XXH_OK;
-}
-
-/*! @ingroup xxh3_family */
-XXH_PUBLIC_API void
-XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed)
-{
- XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
- XXH3_initCustomSecret(secret, seed);
- XXH_ASSERT(secretBuffer != NULL);
- memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE);
-}
-
-
-
-/* Pop our optimization override from above */
-#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
- && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
- && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */
-# pragma GCC pop_options
-#endif
-
-#endif /* XXH_NO_LONG_LONG */
-
-#endif /* XXH_NO_XXH3 */
-
-/*!
- * @}
- */
-#endif /* XXH_IMPLEMENTATION */
-
-
-#if defined (__cplusplus)
-}
-#endif
stage1/zstd/lib/common/zstd_common.c
@@ -1,83 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-
-
-/*-*************************************
-* Dependencies
-***************************************/
-#define ZSTD_DEPS_NEED_MALLOC
-#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
-#include "error_private.h"
-#include "zstd_internal.h"
-
-
-/*-****************************************
-* Version
-******************************************/
-unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; }
-
-const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
-
-
-/*-****************************************
-* ZSTD Error Management
-******************************************/
-#undef ZSTD_isError /* defined within zstd_internal.h */
-/*! ZSTD_isError() :
- * tells if a return value is an error code
- * symbol is required for external callers */
-unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
-
-/*! ZSTD_getErrorName() :
- * provides error code string from function result (useful for debugging) */
-const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
-
-/*! ZSTD_getError() :
- * convert a `size_t` function result into a proper ZSTD_errorCode enum */
-ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
-
-/*! ZSTD_getErrorString() :
- * provides error code string from enum */
-const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
-
-
-
-/*=**************************************************************
-* Custom allocator
-****************************************************************/
-void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
-{
- if (customMem.customAlloc)
- return customMem.customAlloc(customMem.opaque, size);
- return ZSTD_malloc(size);
-}
-
-void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
-{
- if (customMem.customAlloc) {
- /* calloc implemented as malloc+memset;
- * not as efficient as calloc, but next best guess for custom malloc */
- void* const ptr = customMem.customAlloc(customMem.opaque, size);
- ZSTD_memset(ptr, 0, size);
- return ptr;
- }
- return ZSTD_calloc(1, size);
-}
-
-void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
-{
- if (ptr!=NULL) {
- if (customMem.customFree)
- customMem.customFree(customMem.opaque, ptr);
- else
- ZSTD_free(ptr);
- }
-}
stage1/zstd/lib/common/zstd_deps.h
@@ -1,111 +0,0 @@
-/*
- * Copyright (c) Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-/* This file provides common libc dependencies that zstd requires.
- * The purpose is to allow replacing this file with a custom implementation
- * to compile zstd without libc support.
- */
-
-/* Need:
- * NULL
- * INT_MAX
- * UINT_MAX
- * ZSTD_memcpy()
- * ZSTD_memset()
- * ZSTD_memmove()
- */
-#ifndef ZSTD_DEPS_COMMON
-#define ZSTD_DEPS_COMMON
-
-#include <limits.h>
-#include <stddef.h>
-#include <string.h>
-
-#if defined(__GNUC__) && __GNUC__ >= 4
-# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l))
-# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l))
-# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l))
-#else
-# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l))
-# define ZSTD_memmove(d,s,l) memmove((d),(s),(l))
-# define ZSTD_memset(p,v,l) memset((p),(v),(l))
-#endif
-
-#endif /* ZSTD_DEPS_COMMON */
-
-/* Need:
- * ZSTD_malloc()
- * ZSTD_free()
- * ZSTD_calloc()
- */
-#ifdef ZSTD_DEPS_NEED_MALLOC
-#ifndef ZSTD_DEPS_MALLOC
-#define ZSTD_DEPS_MALLOC
-
-#include <stdlib.h>
-
-#define ZSTD_malloc(s) malloc(s)
-#define ZSTD_calloc(n,s) calloc((n), (s))
-#define ZSTD_free(p) free((p))
-
-#endif /* ZSTD_DEPS_MALLOC */
-#endif /* ZSTD_DEPS_NEED_MALLOC */
-
-/*
- * Provides 64-bit math support.
- * Need:
- * U64 ZSTD_div64(U64 dividend, U32 divisor)
- */
-#ifdef ZSTD_DEPS_NEED_MATH64
-#ifndef ZSTD_DEPS_MATH64
-#define ZSTD_DEPS_MATH64
-
-#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor))
-
-#endif /* ZSTD_DEPS_MATH64 */
-#endif /* ZSTD_DEPS_NEED_MATH64 */
-
-/* Need:
- * assert()
- */
-#ifdef ZSTD_DEPS_NEED_ASSERT
-#ifndef ZSTD_DEPS_ASSERT
-#define ZSTD_DEPS_ASSERT
-
-#include <assert.h>
-
-#endif /* ZSTD_DEPS_ASSERT */
-#endif /* ZSTD_DEPS_NEED_ASSERT */
-
-/* Need:
- * ZSTD_DEBUG_PRINT()
- */
-#ifdef ZSTD_DEPS_NEED_IO
-#ifndef ZSTD_DEPS_IO
-#define ZSTD_DEPS_IO
-
-#include <stdio.h>
-#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)
-
-#endif /* ZSTD_DEPS_IO */
-#endif /* ZSTD_DEPS_NEED_IO */
-
-/* Only requested when <stdint.h> is known to be present.
- * Need:
- * intptr_t
- */
-#ifdef ZSTD_DEPS_NEED_STDINT
-#ifndef ZSTD_DEPS_STDINT
-#define ZSTD_DEPS_STDINT
-
-#include <stdint.h>
-
-#endif /* ZSTD_DEPS_STDINT */
-#endif /* ZSTD_DEPS_NEED_STDINT */
stage1/zstd/lib/common/zstd_internal.h
@@ -1,493 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef ZSTD_CCOMMON_H_MODULE
-#define ZSTD_CCOMMON_H_MODULE
-
-/* this module contains definitions which must be identical
- * across compression, decompression and dictBuilder.
- * It also contains a few functions useful to at least 2 of them
- * and which benefit from being inlined */
-
-/*-*************************************
-* Dependencies
-***************************************/
-#include "compiler.h"
-#include "cpu.h"
-#include "mem.h"
-#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */
-#include "error_private.h"
-#define ZSTD_STATIC_LINKING_ONLY
-#include "../zstd.h"
-#define FSE_STATIC_LINKING_ONLY
-#include "fse.h"
-#define HUF_STATIC_LINKING_ONLY
-#include "huf.h"
-#ifndef XXH_STATIC_LINKING_ONLY
-# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
-#endif
-#include "xxhash.h" /* XXH_reset, update, digest */
-#ifndef ZSTD_NO_TRACE
-# include "zstd_trace.h"
-#else
-# define ZSTD_TRACE 0
-#endif
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-/* ---- static assert (debug) --- */
-#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
-#define ZSTD_isError ERR_isError /* for inlining */
-#define FSE_isError ERR_isError
-#define HUF_isError ERR_isError
-
-
-/*-*************************************
-* shared macros
-***************************************/
-#undef MIN
-#undef MAX
-#define MIN(a,b) ((a)<(b) ? (a) : (b))
-#define MAX(a,b) ((a)>(b) ? (a) : (b))
-#define BOUNDED(min,val,max) (MAX(min,MIN(val,max)))
-
-
-/*-*************************************
-* Common constants
-***************************************/
-#define ZSTD_OPT_NUM (1<<12)
-
-#define ZSTD_REP_NUM 3 /* number of repcodes */
-static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
-
-#define KB *(1 <<10)
-#define MB *(1 <<20)
-#define GB *(1U<<30)
-
-#define BIT7 128
-#define BIT6 64
-#define BIT5 32
-#define BIT4 16
-#define BIT1 2
-#define BIT0 1
-
-#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
-static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
-static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
-
-#define ZSTD_FRAMEIDSIZE 4 /* magic number size */
-
-#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
-static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
-typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
-
-#define ZSTD_FRAMECHECKSUMSIZE 4
-
-#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
-#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
-
-#define HufLog 12
-typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
-
-#define LONGNBSEQ 0x7F00
-
-#define MINMATCH 3
-
-#define Litbits 8
-#define MaxLit ((1<<Litbits) - 1)
-#define MaxML 52
-#define MaxLL 35
-#define DefaultMaxOff 28
-#define MaxOff 31
-#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
-#define MLFSELog 9
-#define LLFSELog 9
-#define OffFSELog 8
-#define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
-
-#define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */
-/* Each table cannot take more than #symbols * FSELog bits */
-#define ZSTD_MAX_FSE_HEADERS_SIZE (((MaxML + 1) * MLFSELog + (MaxLL + 1) * LLFSELog + (MaxOff + 1) * OffFSELog + 7) / 8)
-
-static UNUSED_ATTR const U8 LL_bits[MaxLL+1] = {
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 1, 1, 1, 1, 2, 2, 3, 3,
- 4, 6, 7, 8, 9,10,11,12,
- 13,14,15,16
-};
-static UNUSED_ATTR const S16 LL_defaultNorm[MaxLL+1] = {
- 4, 3, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 1, 1, 1,
- 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 3, 2, 1, 1, 1, 1, 1,
- -1,-1,-1,-1
-};
-#define LL_DEFAULTNORMLOG 6 /* for static allocation */
-static UNUSED_ATTR const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
-
-static UNUSED_ATTR const U8 ML_bits[MaxML+1] = {
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 1, 1, 1, 1, 2, 2, 3, 3,
- 4, 4, 5, 7, 8, 9,10,11,
- 12,13,14,15,16
-};
-static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = {
- 1, 4, 3, 2, 2, 2, 2, 2,
- 2, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1,-1,-1,
- -1,-1,-1,-1,-1
-};
-#define ML_DEFAULTNORMLOG 6 /* for static allocation */
-static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
-
-static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = {
- 1, 1, 1, 1, 1, 1, 2, 2,
- 2, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1,
- -1,-1,-1,-1,-1
-};
-#define OF_DEFAULTNORMLOG 5 /* for static allocation */
-static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
-
-
-/*-*******************************************
-* Shared functions to include for inlining
-*********************************************/
-static void ZSTD_copy8(void* dst, const void* src) {
-#if defined(ZSTD_ARCH_ARM_NEON)
- vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src));
-#else
- ZSTD_memcpy(dst, src, 8);
-#endif
-}
-#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
-
-/* Need to use memmove here since the literal buffer can now be located within
- the dst buffer. In circumstances where the op "catches up" to where the
- literal buffer is, there can be partial overlaps in this call on the final
- copy if the literal is being shifted by less than 16 bytes. */
-static void ZSTD_copy16(void* dst, const void* src) {
-#if defined(ZSTD_ARCH_ARM_NEON)
- vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src));
-#elif defined(ZSTD_ARCH_X86_SSE2)
- _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src));
-#elif defined(__clang__)
- ZSTD_memmove(dst, src, 16);
-#else
- /* ZSTD_memmove is not inlined properly by gcc */
- BYTE copy16_buf[16];
- ZSTD_memcpy(copy16_buf, src, 16);
- ZSTD_memcpy(dst, copy16_buf, 16);
-#endif
-}
-#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
-
-#define WILDCOPY_OVERLENGTH 32
-#define WILDCOPY_VECLEN 16
-
-typedef enum {
- ZSTD_no_overlap,
- ZSTD_overlap_src_before_dst
- /* ZSTD_overlap_dst_before_src, */
-} ZSTD_overlap_e;
-
-/*! ZSTD_wildcopy() :
- * Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0)
- * @param ovtype controls the overlap detection
- * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
- * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart.
- * The src buffer must be before the dst buffer.
- */
-MEM_STATIC FORCE_INLINE_ATTR
-void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype)
-{
- ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
- const BYTE* ip = (const BYTE*)src;
- BYTE* op = (BYTE*)dst;
- BYTE* const oend = op + length;
-
- if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) {
- /* Handle short offset copies. */
- do {
- COPY8(op, ip)
- } while (op < oend);
- } else {
- assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN);
- /* Separate out the first COPY16() call because the copy length is
- * almost certain to be short, so the branches have different
- * probabilities. Since it is almost certain to be short, only do
- * one COPY16() in the first call. Then, do two calls per loop since
- * at that point it is more likely to have a high trip count.
- */
-#ifdef __aarch64__
- do {
- COPY16(op, ip);
- }
- while (op < oend);
-#else
- ZSTD_copy16(op, ip);
- if (16 >= length) return;
- op += 16;
- ip += 16;
- do {
- COPY16(op, ip);
- COPY16(op, ip);
- }
- while (op < oend);
-#endif
- }
-}
-
-MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
-{
- size_t const length = MIN(dstCapacity, srcSize);
- if (length > 0) {
- ZSTD_memcpy(dst, src, length);
- }
- return length;
-}
-
-/* define "workspace is too large" as this number of times larger than needed */
-#define ZSTD_WORKSPACETOOLARGE_FACTOR 3
-
-/* when workspace is continuously too large
- * during at least this number of times,
- * context's memory usage is considered wasteful,
- * because it's sized to handle a worst case scenario which rarely happens.
- * In which case, resize it down to free some memory */
-#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
-
-/* Controls whether the input/output buffer is buffered or stable. */
-typedef enum {
- ZSTD_bm_buffered = 0, /* Buffer the input/output */
- ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */
-} ZSTD_bufferMode_e;
-
-
-/*-*******************************************
-* Private declarations
-*********************************************/
-typedef struct seqDef_s {
- U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
- U16 litLength;
- U16 mlBase; /* mlBase == matchLength - MINMATCH */
-} seqDef;
-
-/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
-typedef enum {
- ZSTD_llt_none = 0, /* no longLengthType */
- ZSTD_llt_literalLength = 1, /* represents a long literal */
- ZSTD_llt_matchLength = 2 /* represents a long match */
-} ZSTD_longLengthType_e;
-
-typedef struct {
- seqDef* sequencesStart;
- seqDef* sequences; /* ptr to end of sequences */
- BYTE* litStart;
- BYTE* lit; /* ptr to end of literals */
- BYTE* llCode;
- BYTE* mlCode;
- BYTE* ofCode;
- size_t maxNbSeq;
- size_t maxNbLit;
-
- /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
- * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
- * the existing value of the litLength or matchLength by 0x10000.
- */
- ZSTD_longLengthType_e longLengthType;
- U32 longLengthPos; /* Index of the sequence to apply long length modification to */
-} seqStore_t;
-
-typedef struct {
- U32 litLength;
- U32 matchLength;
-} ZSTD_sequenceLength;
-
-/**
- * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences
- * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
- */
-MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq)
-{
- ZSTD_sequenceLength seqLen;
- seqLen.litLength = seq->litLength;
- seqLen.matchLength = seq->mlBase + MINMATCH;
- if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
- if (seqStore->longLengthType == ZSTD_llt_literalLength) {
- seqLen.litLength += 0xFFFF;
- }
- if (seqStore->longLengthType == ZSTD_llt_matchLength) {
- seqLen.matchLength += 0xFFFF;
- }
- }
- return seqLen;
-}
-
-/**
- * Contains the compressed frame size and an upper-bound for the decompressed frame size.
- * Note: before using `compressedSize`, check for errors using ZSTD_isError().
- * similarly, before using `decompressedBound`, check for errors using:
- * `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
- */
-typedef struct {
- size_t compressedSize;
- unsigned long long decompressedBound;
-} ZSTD_frameSizeInfo; /* decompress & legacy */
-
-const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
-void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
-
-/* custom memory allocation functions */
-void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);
-void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem);
-void ZSTD_customFree(void* ptr, ZSTD_customMem customMem);
-
-
-MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */
-{
- assert(val != 0);
- {
-# if defined(_MSC_VER) /* Visual */
-# if STATIC_BMI2 == 1
- return _lzcnt_u32(val)^31;
-# else
- if (val != 0) {
- unsigned long r;
- _BitScanReverse(&r, val);
- return (unsigned)r;
- } else {
- /* Should not reach this code path */
- __assume(0);
- }
-# endif
-# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
- return __builtin_clz (val) ^ 31;
-# elif defined(__ICCARM__) /* IAR Intrinsic */
- return 31 - __CLZ(val);
-# else /* Software version */
- static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
- U32 v = val;
- v |= v >> 1;
- v |= v >> 2;
- v |= v >> 4;
- v |= v >> 8;
- v |= v >> 16;
- return DeBruijnClz[(v * 0x07C4ACDDU) >> 27];
-# endif
- }
-}
-
-/**
- * Counts the number of trailing zeros of a `size_t`.
- * Most compilers should support CTZ as a builtin. A backup
- * implementation is provided if the builtin isn't supported, but
- * it may not be terribly efficient.
- */
-MEM_STATIC unsigned ZSTD_countTrailingZeros(size_t val)
-{
- if (MEM_64bits()) {
-# if defined(_MSC_VER) && defined(_WIN64)
-# if STATIC_BMI2
- return _tzcnt_u64(val);
-# else
- if (val != 0) {
- unsigned long r;
- _BitScanForward64(&r, (U64)val);
- return (unsigned)r;
- } else {
- /* Should not reach this code path */
- __assume(0);
- }
-# endif
-# elif defined(__GNUC__) && (__GNUC__ >= 4)
- return __builtin_ctzll((U64)val);
-# else
- static const int DeBruijnBytePos[64] = { 0, 1, 2, 7, 3, 13, 8, 19,
- 4, 25, 14, 28, 9, 34, 20, 56,
- 5, 17, 26, 54, 15, 41, 29, 43,
- 10, 31, 38, 35, 21, 45, 49, 57,
- 63, 6, 12, 18, 24, 27, 33, 55,
- 16, 53, 40, 42, 30, 37, 44, 48,
- 62, 11, 23, 32, 52, 39, 36, 47,
- 61, 22, 51, 46, 60, 50, 59, 58 };
- return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-# endif
- } else { /* 32 bits */
-# if defined(_MSC_VER)
- if (val != 0) {
- unsigned long r;
- _BitScanForward(&r, (U32)val);
- return (unsigned)r;
- } else {
- /* Should not reach this code path */
- __assume(0);
- }
-# elif defined(__GNUC__) && (__GNUC__ >= 3)
- return __builtin_ctz((U32)val);
-# else
- static const int DeBruijnBytePos[32] = { 0, 1, 28, 2, 29, 14, 24, 3,
- 30, 22, 20, 15, 25, 17, 4, 8,
- 31, 27, 13, 23, 21, 19, 16, 7,
- 26, 12, 18, 6, 11, 5, 10, 9 };
- return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-# endif
- }
-}
-
-
-/* ZSTD_invalidateRepCodes() :
- * ensures next compression will not use repcodes from previous block.
- * Note : only works with regular variant;
- * do not use with extDict variant ! */
-void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */
-
-
-typedef struct {
- blockType_e blockType;
- U32 lastBlock;
- U32 origSize;
-} blockProperties_t; /* declared here for decompress and fullbench */
-
-/*! ZSTD_getcBlockSize() :
- * Provides the size of compressed block from block header `src` */
-/* Used by: decompress, fullbench (does not get its definition from here) */
-size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
- blockProperties_t* bpPtr);
-
-/*! ZSTD_decodeSeqHeaders() :
- * decode sequence header from src */
-/* Used by: decompress, fullbench (does not get its definition from here) */
-size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
- const void* src, size_t srcSize);
-
-/**
- * @returns true iff the CPU supports dynamic BMI2 dispatch.
- */
-MEM_STATIC int ZSTD_cpuSupportsBmi2(void)
-{
- ZSTD_cpuid_t cpuid = ZSTD_cpuid();
- return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid);
-}
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ZSTD_CCOMMON_H_MODULE */
stage1/zstd/lib/common/zstd_trace.h
@@ -1,163 +0,0 @@
-/*
- * Copyright (c) Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef ZSTD_TRACE_H
-#define ZSTD_TRACE_H
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-#include <stddef.h>
-
-/* weak symbol support
- * For now, enable conservatively:
- * - Only GNUC
- * - Only ELF
- * - Only x86-64 and i386
- * Also, explicitly disable on platforms known not to work so they aren't
- * forgotten in the future.
- */
-#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \
- defined(__GNUC__) && defined(__ELF__) && \
- (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)) && \
- !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
- !defined(__CYGWIN__) && !defined(_AIX)
-# define ZSTD_HAVE_WEAK_SYMBOLS 1
-#else
-# define ZSTD_HAVE_WEAK_SYMBOLS 0
-#endif
-#if ZSTD_HAVE_WEAK_SYMBOLS
-# define ZSTD_WEAK_ATTR __attribute__((__weak__))
-#else
-# define ZSTD_WEAK_ATTR
-#endif
-
-/* Only enable tracing when weak symbols are available. */
-#ifndef ZSTD_TRACE
-# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
-#endif
-
-#if ZSTD_TRACE
-
-struct ZSTD_CCtx_s;
-struct ZSTD_DCtx_s;
-struct ZSTD_CCtx_params_s;
-
-typedef struct {
- /**
- * ZSTD_VERSION_NUMBER
- *
- * This is guaranteed to be the first member of ZSTD_trace.
- * Otherwise, this struct is not stable between versions. If
- * the version number does not match your expectation, you
- * should not interpret the rest of the struct.
- */
- unsigned version;
- /**
- * Non-zero if streaming (de)compression is used.
- */
- unsigned streaming;
- /**
- * The dictionary ID.
- */
- unsigned dictionaryID;
- /**
- * Is the dictionary cold?
- * Only set on decompression.
- */
- unsigned dictionaryIsCold;
- /**
- * The dictionary size or zero if no dictionary.
- */
- size_t dictionarySize;
- /**
- * The uncompressed size of the data.
- */
- size_t uncompressedSize;
- /**
- * The compressed size of the data.
- */
- size_t compressedSize;
- /**
- * The fully resolved CCtx parameters (NULL on decompression).
- */
- struct ZSTD_CCtx_params_s const* params;
- /**
- * The ZSTD_CCtx pointer (NULL on decompression).
- */
- struct ZSTD_CCtx_s const* cctx;
- /**
- * The ZSTD_DCtx pointer (NULL on compression).
- */
- struct ZSTD_DCtx_s const* dctx;
-} ZSTD_Trace;
-
-/**
- * A tracing context. It must be 0 when tracing is disabled.
- * Otherwise, any non-zero value returned by a tracing begin()
- * function is presented to any subsequent calls to end().
- *
- * Any non-zero value is treated as tracing is enabled and not
- * interpreted by the library.
- *
- * Two possible uses are:
- * * A timestamp for when the begin() function was called.
- * * A unique key identifying the (de)compression, like the
- * address of the [dc]ctx pointer if you need to track
- * more information than just a timestamp.
- */
-typedef unsigned long long ZSTD_TraceCtx;
-
-/**
- * Trace the beginning of a compression call.
- * @param cctx The dctx pointer for the compression.
- * It can be used as a key to map begin() to end().
- * @returns Non-zero if tracing is enabled. The return value is
- * passed to ZSTD_trace_compress_end().
- */
-ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(
- struct ZSTD_CCtx_s const* cctx);
-
-/**
- * Trace the end of a compression call.
- * @param ctx The return value of ZSTD_trace_compress_begin().
- * @param trace The zstd tracing info.
- */
-ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(
- ZSTD_TraceCtx ctx,
- ZSTD_Trace const* trace);
-
-/**
- * Trace the beginning of a decompression call.
- * @param dctx The dctx pointer for the decompression.
- * It can be used as a key to map begin() to end().
- * @returns Non-zero if tracing is enabled. The return value is
- * passed to ZSTD_trace_compress_end().
- */
-ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(
- struct ZSTD_DCtx_s const* dctx);
-
-/**
- * Trace the end of a decompression call.
- * @param ctx The return value of ZSTD_trace_decompress_begin().
- * @param trace The zstd tracing info.
- */
-ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
- ZSTD_TraceCtx ctx,
- ZSTD_Trace const* trace);
-
-#endif /* ZSTD_TRACE */
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ZSTD_TRACE_H */
stage1/zstd/lib/decompress/huf_decompress.c
@@ -1,1889 +0,0 @@
-/* ******************************************************************
- * huff0 huffman decoder,
- * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
- *
- * You can contact the author at :
- * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
-****************************************************************** */
-
-/* **************************************************************
-* Dependencies
-****************************************************************/
-#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */
-#include "../common/compiler.h"
-#include "../common/bitstream.h" /* BIT_* */
-#include "../common/fse.h" /* to compress headers */
-#define HUF_STATIC_LINKING_ONLY
-#include "../common/huf.h"
-#include "../common/error_private.h"
-#include "../common/zstd_internal.h"
-
-/* **************************************************************
-* Constants
-****************************************************************/
-
-#define HUF_DECODER_FAST_TABLELOG 11
-
-/* **************************************************************
-* Macros
-****************************************************************/
-
-/* These two optional macros force the use one way or another of the two
- * Huffman decompression implementations. You can't force in both directions
- * at the same time.
- */
-#if defined(HUF_FORCE_DECOMPRESS_X1) && \
- defined(HUF_FORCE_DECOMPRESS_X2)
-#error "Cannot force the use of the X1 and X2 decoders at the same time!"
-#endif
-
-#if ZSTD_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2
-# define HUF_ASM_X86_64_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
-#else
-# define HUF_ASM_X86_64_BMI2_ATTRS
-#endif
-
-#ifdef __cplusplus
-# define HUF_EXTERN_C extern "C"
-#else
-# define HUF_EXTERN_C
-#endif
-#define HUF_ASM_DECL HUF_EXTERN_C
-
-#if DYNAMIC_BMI2 || (ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
-# define HUF_NEED_BMI2_FUNCTION 1
-#else
-# define HUF_NEED_BMI2_FUNCTION 0
-#endif
-
-#if !(ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
-# define HUF_NEED_DEFAULT_FUNCTION 1
-#else
-# define HUF_NEED_DEFAULT_FUNCTION 0
-#endif
-
-/* **************************************************************
-* Error Management
-****************************************************************/
-#define HUF_isError ERR_isError
-
-
-/* **************************************************************
-* Byte alignment for workSpace management
-****************************************************************/
-#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1)
-#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
-
-
-/* **************************************************************
-* BMI2 Variant Wrappers
-****************************************************************/
-#if DYNAMIC_BMI2
-
-#define HUF_DGEN(fn) \
- \
- static size_t fn##_default( \
- void* dst, size_t dstSize, \
- const void* cSrc, size_t cSrcSize, \
- const HUF_DTable* DTable) \
- { \
- return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
- } \
- \
- static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \
- void* dst, size_t dstSize, \
- const void* cSrc, size_t cSrcSize, \
- const HUF_DTable* DTable) \
- { \
- return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
- } \
- \
- static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
- size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
- { \
- if (bmi2) { \
- return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \
- } \
- return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \
- }
-
-#else
-
-#define HUF_DGEN(fn) \
- static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
- size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
- { \
- (void)bmi2; \
- return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
- }
-
-#endif
-
-
-/*-***************************/
-/* generic DTableDesc */
-/*-***************************/
-typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;
-
-static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
-{
- DTableDesc dtd;
- ZSTD_memcpy(&dtd, table, sizeof(dtd));
- return dtd;
-}
-
-#if ZSTD_ENABLE_ASM_X86_64_BMI2
-
-static size_t HUF_initDStream(BYTE const* ip) {
- BYTE const lastByte = ip[7];
- size_t const bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
- size_t const value = MEM_readLEST(ip) | 1;
- assert(bitsConsumed <= 8);
- return value << bitsConsumed;
-}
-typedef struct {
- BYTE const* ip[4];
- BYTE* op[4];
- U64 bits[4];
- void const* dt;
- BYTE const* ilimit;
- BYTE* oend;
- BYTE const* iend[4];
-} HUF_DecompressAsmArgs;
-
-/**
- * Initializes args for the asm decoding loop.
- * @returns 0 on success
- * 1 if the fallback implementation should be used.
- * Or an error code on failure.
- */
-static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
-{
- void const* dt = DTable + 1;
- U32 const dtLog = HUF_getDTableDesc(DTable).tableLog;
-
- const BYTE* const ilimit = (const BYTE*)src + 6 + 8;
-
- BYTE* const oend = (BYTE*)dst + dstSize;
-
- /* The following condition is false on x32 platform,
- * but HUF_asm is not compatible with this ABI */
- if (!(MEM_isLittleEndian() && !MEM_32bits())) return 1;
-
- /* strict minimum : jump table + 1 byte per stream */
- if (srcSize < 10)
- return ERROR(corruption_detected);
-
- /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers.
- * If table log is not correct at this point, fallback to the old decoder.
- * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder.
- */
- if (dtLog != HUF_DECODER_FAST_TABLELOG)
- return 1;
-
- /* Read the jump table. */
- {
- const BYTE* const istart = (const BYTE*)src;
- size_t const length1 = MEM_readLE16(istart);
- size_t const length2 = MEM_readLE16(istart+2);
- size_t const length3 = MEM_readLE16(istart+4);
- size_t const length4 = srcSize - (length1 + length2 + length3 + 6);
- args->iend[0] = istart + 6; /* jumpTable */
- args->iend[1] = args->iend[0] + length1;
- args->iend[2] = args->iend[1] + length2;
- args->iend[3] = args->iend[2] + length3;
-
- /* HUF_initDStream() requires this, and this small of an input
- * won't benefit from the ASM loop anyways.
- * length1 must be >= 16 so that ip[0] >= ilimit before the loop
- * starts.
- */
- if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8)
- return 1;
- if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */
- }
- /* ip[] contains the position that is currently loaded into bits[]. */
- args->ip[0] = args->iend[1] - sizeof(U64);
- args->ip[1] = args->iend[2] - sizeof(U64);
- args->ip[2] = args->iend[3] - sizeof(U64);
- args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64);
-
- /* op[] contains the output pointers. */
- args->op[0] = (BYTE*)dst;
- args->op[1] = args->op[0] + (dstSize+3)/4;
- args->op[2] = args->op[1] + (dstSize+3)/4;
- args->op[3] = args->op[2] + (dstSize+3)/4;
-
- /* No point to call the ASM loop for tiny outputs. */
- if (args->op[3] >= oend)
- return 1;
-
- /* bits[] is the bit container.
- * It is read from the MSB down to the LSB.
- * It is shifted left as it is read, and zeros are
- * shifted in. After the lowest valid bit a 1 is
- * set, so that CountTrailingZeros(bits[]) can be used
- * to count how many bits we've consumed.
- */
- args->bits[0] = HUF_initDStream(args->ip[0]);
- args->bits[1] = HUF_initDStream(args->ip[1]);
- args->bits[2] = HUF_initDStream(args->ip[2]);
- args->bits[3] = HUF_initDStream(args->ip[3]);
-
- /* If ip[] >= ilimit, it is guaranteed to be safe to
- * reload bits[]. It may be beyond its section, but is
- * guaranteed to be valid (>= istart).
- */
- args->ilimit = ilimit;
-
- args->oend = oend;
- args->dt = dt;
-
- return 0;
-}
-
-static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs const* args, int stream, BYTE* segmentEnd)
-{
- /* Validate that we haven't overwritten. */
- if (args->op[stream] > segmentEnd)
- return ERROR(corruption_detected);
- /* Validate that we haven't read beyond iend[].
- * Note that ip[] may be < iend[] because the MSB is
- * the next bit to read, and we may have consumed 100%
- * of the stream, so down to iend[i] - 8 is valid.
- */
- if (args->ip[stream] < args->iend[stream] - 8)
- return ERROR(corruption_detected);
-
- /* Construct the BIT_DStream_t. */
- bit->bitContainer = MEM_readLE64(args->ip[stream]);
- bit->bitsConsumed = ZSTD_countTrailingZeros((size_t)args->bits[stream]);
- bit->start = (const char*)args->iend[0];
- bit->limitPtr = bit->start + sizeof(size_t);
- bit->ptr = (const char*)args->ip[stream];
-
- return 0;
-}
-#endif
-
-
-#ifndef HUF_FORCE_DECOMPRESS_X2
-
-/*-***************************/
-/* single-symbol decoding */
-/*-***************************/
-typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */
-
-/**
- * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at
- * a time.
- */
-static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) {
- U64 D4;
- if (MEM_isLittleEndian()) {
- D4 = (symbol << 8) + nbBits;
- } else {
- D4 = symbol + (nbBits << 8);
- }
- D4 *= 0x0001000100010001ULL;
- return D4;
-}
-
-/**
- * Increase the tableLog to targetTableLog and rescales the stats.
- * If tableLog > targetTableLog this is a no-op.
- * @returns New tableLog
- */
-static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog)
-{
- if (tableLog > targetTableLog)
- return tableLog;
- if (tableLog < targetTableLog) {
- U32 const scale = targetTableLog - tableLog;
- U32 s;
- /* Increase the weight for all non-zero probability symbols by scale. */
- for (s = 0; s < nbSymbols; ++s) {
- huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale);
- }
- /* Update rankVal to reflect the new weights.
- * All weights except 0 get moved to weight + scale.
- * Weights [1, scale] are empty.
- */
- for (s = targetTableLog; s > scale; --s) {
- rankVal[s] = rankVal[s - scale];
- }
- for (s = scale; s > 0; --s) {
- rankVal[s] = 0;
- }
- }
- return targetTableLog;
-}
-
-typedef struct {
- U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
- U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1];
- U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
- BYTE symbols[HUF_SYMBOLVALUE_MAX + 1];
- BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
-} HUF_ReadDTableX1_Workspace;
-
-
-size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize)
-{
- return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
-size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2)
-{
- U32 tableLog = 0;
- U32 nbSymbols = 0;
- size_t iSize;
- void* const dtPtr = DTable + 1;
- HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr;
- HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace;
-
- DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp));
- if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge);
-
- DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
- /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
-
- iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2);
- if (HUF_isError(iSize)) return iSize;
-
-
- /* Table header */
- { DTableDesc dtd = HUF_getDTableDesc(DTable);
- U32 const maxTableLog = dtd.maxTableLog + 1;
- U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG);
- tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog);
- if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */
- dtd.tableType = 0;
- dtd.tableLog = (BYTE)tableLog;
- ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
- }
-
- /* Compute symbols and rankStart given rankVal:
- *
- * rankVal already contains the number of values of each weight.
- *
- * symbols contains the symbols ordered by weight. First are the rankVal[0]
- * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on.
- * symbols[0] is filled (but unused) to avoid a branch.
- *
- * rankStart contains the offset where each rank belongs in the DTable.
- * rankStart[0] is not filled because there are no entries in the table for
- * weight 0.
- */
- {
- int n;
- int nextRankStart = 0;
- int const unroll = 4;
- int const nLimit = (int)nbSymbols - unroll + 1;
- for (n=0; n<(int)tableLog+1; n++) {
- U32 const curr = nextRankStart;
- nextRankStart += wksp->rankVal[n];
- wksp->rankStart[n] = curr;
- }
- for (n=0; n < nLimit; n += unroll) {
- int u;
- for (u=0; u < unroll; ++u) {
- size_t const w = wksp->huffWeight[n+u];
- wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u);
- }
- }
- for (; n < (int)nbSymbols; ++n) {
- size_t const w = wksp->huffWeight[n];
- wksp->symbols[wksp->rankStart[w]++] = (BYTE)n;
- }
- }
-
- /* fill DTable
- * We fill all entries of each weight in order.
- * That way length is a constant for each iteration of the outer loop.
- * We can switch based on the length to a different inner loop which is
- * optimized for that particular case.
- */
- {
- U32 w;
- int symbol=wksp->rankVal[0];
- int rankStart=0;
- for (w=1; w<tableLog+1; ++w) {
- int const symbolCount = wksp->rankVal[w];
- int const length = (1 << w) >> 1;
- int uStart = rankStart;
- BYTE const nbBits = (BYTE)(tableLog + 1 - w);
- int s;
- int u;
- switch (length) {
- case 1:
- for (s=0; s<symbolCount; ++s) {
- HUF_DEltX1 D;
- D.byte = wksp->symbols[symbol + s];
- D.nbBits = nbBits;
- dt[uStart] = D;
- uStart += 1;
- }
- break;
- case 2:
- for (s=0; s<symbolCount; ++s) {
- HUF_DEltX1 D;
- D.byte = wksp->symbols[symbol + s];
- D.nbBits = nbBits;
- dt[uStart+0] = D;
- dt[uStart+1] = D;
- uStart += 2;
- }
- break;
- case 4:
- for (s=0; s<symbolCount; ++s) {
- U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
- MEM_write64(dt + uStart, D4);
- uStart += 4;
- }
- break;
- case 8:
- for (s=0; s<symbolCount; ++s) {
- U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
- MEM_write64(dt + uStart, D4);
- MEM_write64(dt + uStart + 4, D4);
- uStart += 8;
- }
- break;
- default:
- for (s=0; s<symbolCount; ++s) {
- U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
- for (u=0; u < length; u += 16) {
- MEM_write64(dt + uStart + u + 0, D4);
- MEM_write64(dt + uStart + u + 4, D4);
- MEM_write64(dt + uStart + u + 8, D4);
- MEM_write64(dt + uStart + u + 12, D4);
- }
- assert(u == length);
- uStart += length;
- }
- break;
- }
- symbol += symbolCount;
- rankStart += symbolCount * length;
- }
- }
- return iSize;
-}
-
-FORCE_INLINE_TEMPLATE BYTE
-HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog)
-{
- size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
- BYTE const c = dt[val].byte;
- BIT_skipBits(Dstream, dt[val].nbBits);
- return c;
-}
-
-#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \
- *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog)
-
-#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \
- if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
- HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr)
-
-#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \
- if (MEM_64bits()) \
- HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr)
-
-HINT_INLINE size_t
-HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog)
-{
- BYTE* const pStart = p;
-
- /* up to 4 symbols at a time */
- if ((pEnd - p) > 3) {
- while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) {
- HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
- HUF_DECODE_SYMBOLX1_1(p, bitDPtr);
- HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
- HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
- }
- } else {
- BIT_reloadDStream(bitDPtr);
- }
-
- /* [0-3] symbols remaining */
- if (MEM_32bits())
- while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd))
- HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
-
- /* no more data to retrieve from bitstream, no need to reload */
- while (p < pEnd)
- HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
-
- return pEnd-pStart;
-}
-
-FORCE_INLINE_TEMPLATE size_t
-HUF_decompress1X1_usingDTable_internal_body(
- void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable)
-{
- BYTE* op = (BYTE*)dst;
- BYTE* const oend = op + dstSize;
- const void* dtPtr = DTable + 1;
- const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
- BIT_DStream_t bitD;
- DTableDesc const dtd = HUF_getDTableDesc(DTable);
- U32 const dtLog = dtd.tableLog;
-
- CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
-
- HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog);
-
- if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
-
- return dstSize;
-}
-
-FORCE_INLINE_TEMPLATE size_t
-HUF_decompress4X1_usingDTable_internal_body(
- void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable)
-{
- /* Check */
- if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
-
- { const BYTE* const istart = (const BYTE*) cSrc;
- BYTE* const ostart = (BYTE*) dst;
- BYTE* const oend = ostart + dstSize;
- BYTE* const olimit = oend - 3;
- const void* const dtPtr = DTable + 1;
- const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
-
- /* Init */
- BIT_DStream_t bitD1;
- BIT_DStream_t bitD2;
- BIT_DStream_t bitD3;
- BIT_DStream_t bitD4;
- size_t const length1 = MEM_readLE16(istart);
- size_t const length2 = MEM_readLE16(istart+2);
- size_t const length3 = MEM_readLE16(istart+4);
- size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
- const BYTE* const istart1 = istart + 6; /* jumpTable */
- const BYTE* const istart2 = istart1 + length1;
- const BYTE* const istart3 = istart2 + length2;
- const BYTE* const istart4 = istart3 + length3;
- const size_t segmentSize = (dstSize+3) / 4;
- BYTE* const opStart2 = ostart + segmentSize;
- BYTE* const opStart3 = opStart2 + segmentSize;
- BYTE* const opStart4 = opStart3 + segmentSize;
- BYTE* op1 = ostart;
- BYTE* op2 = opStart2;
- BYTE* op3 = opStart3;
- BYTE* op4 = opStart4;
- DTableDesc const dtd = HUF_getDTableDesc(DTable);
- U32 const dtLog = dtd.tableLog;
- U32 endSignal = 1;
-
- if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
- if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */
- CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
- CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
- CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
- CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
-
- /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */
- if ((size_t)(oend - op4) >= sizeof(size_t)) {
- for ( ; (endSignal) & (op4 < olimit) ; ) {
- HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
- HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
- HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
- HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
- HUF_DECODE_SYMBOLX1_1(op1, &bitD1);
- HUF_DECODE_SYMBOLX1_1(op2, &bitD2);
- HUF_DECODE_SYMBOLX1_1(op3, &bitD3);
- HUF_DECODE_SYMBOLX1_1(op4, &bitD4);
- HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
- HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
- HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
- HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
- HUF_DECODE_SYMBOLX1_0(op1, &bitD1);
- HUF_DECODE_SYMBOLX1_0(op2, &bitD2);
- HUF_DECODE_SYMBOLX1_0(op3, &bitD3);
- HUF_DECODE_SYMBOLX1_0(op4, &bitD4);
- endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
- endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
- endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
- endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
- }
- }
-
- /* check corruption */
- /* note : should not be necessary : op# advance in lock step, and we control op4.
- * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */
- if (op1 > opStart2) return ERROR(corruption_detected);
- if (op2 > opStart3) return ERROR(corruption_detected);
- if (op3 > opStart4) return ERROR(corruption_detected);
- /* note : op4 supposed already verified within main loop */
-
- /* finish bitStreams one by one */
- HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog);
- HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog);
- HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog);
- HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog);
-
- /* check */
- { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
- if (!endCheck) return ERROR(corruption_detected); }
-
- /* decoded size */
- return dstSize;
- }
-}
-
-#if HUF_NEED_BMI2_FUNCTION
-static BMI2_TARGET_ATTRIBUTE
-size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
- size_t cSrcSize, HUF_DTable const* DTable) {
- return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
-}
-#endif
-
-#if HUF_NEED_DEFAULT_FUNCTION
-static
-size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
- size_t cSrcSize, HUF_DTable const* DTable) {
- return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
-}
-#endif
-
-#if ZSTD_ENABLE_ASM_X86_64_BMI2
-
-HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN;
-
-static HUF_ASM_X86_64_BMI2_ATTRS
-size_t
-HUF_decompress4X1_usingDTable_internal_bmi2_asm(
- void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable)
-{
- void const* dt = DTable + 1;
- const BYTE* const iend = (const BYTE*)cSrc + 6;
- BYTE* const oend = (BYTE*)dst + dstSize;
- HUF_DecompressAsmArgs args;
- {
- size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
- FORWARD_IF_ERROR(ret, "Failed to init asm args");
- if (ret != 0)
- return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
- }
-
- assert(args.ip[0] >= args.ilimit);
- HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(&args);
-
- /* Our loop guarantees that ip[] >= ilimit and that we haven't
- * overwritten any op[].
- */
- assert(args.ip[0] >= iend);
- assert(args.ip[1] >= iend);
- assert(args.ip[2] >= iend);
- assert(args.ip[3] >= iend);
- assert(args.op[3] <= oend);
- (void)iend;
-
- /* finish bit streams one by one. */
- {
- size_t const segmentSize = (dstSize+3) / 4;
- BYTE* segmentEnd = (BYTE*)dst;
- int i;
- for (i = 0; i < 4; ++i) {
- BIT_DStream_t bit;
- if (segmentSize <= (size_t)(oend - segmentEnd))
- segmentEnd += segmentSize;
- else
- segmentEnd = oend;
- FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
- /* Decompress and validate that we've produced exactly the expected length. */
- args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG);
- if (args.op[i] != segmentEnd) return ERROR(corruption_detected);
- }
- }
-
- /* decoded size */
- return dstSize;
-}
-#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
-
-typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize,
- const void *cSrc,
- size_t cSrcSize,
- const HUF_DTable *DTable);
-
-HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
-
-static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
- size_t cSrcSize, HUF_DTable const* DTable, int bmi2)
-{
-#if DYNAMIC_BMI2
- if (bmi2) {
-# if ZSTD_ENABLE_ASM_X86_64_BMI2
- return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-# else
- return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
-# endif
- }
-#else
- (void)bmi2;
-#endif
-
-#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
- return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-#else
- return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
-#endif
-}
-
-
-size_t HUF_decompress1X1_usingDTable(
- void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable)
-{
- DTableDesc dtd = HUF_getDTableDesc(DTable);
- if (dtd.tableType != 0) return ERROR(GENERIC);
- return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
-size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- void* workSpace, size_t wkspSize)
-{
- const BYTE* ip = (const BYTE*) cSrc;
-
- size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize);
- if (HUF_isError(hSize)) return hSize;
- if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
- ip += hSize; cSrcSize -= hSize;
-
- return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0);
-}
-
-
-size_t HUF_decompress4X1_usingDTable(
- void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable)
-{
- DTableDesc dtd = HUF_getDTableDesc(DTable);
- if (dtd.tableType != 0) return ERROR(GENERIC);
- return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
-static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- void* workSpace, size_t wkspSize, int bmi2)
-{
- const BYTE* ip = (const BYTE*) cSrc;
-
- size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
- if (HUF_isError(hSize)) return hSize;
- if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
- ip += hSize; cSrcSize -= hSize;
-
- return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
-}
-
-size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- void* workSpace, size_t wkspSize)
-{
- return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0);
-}
-
-
-#endif /* HUF_FORCE_DECOMPRESS_X2 */
-
-
-#ifndef HUF_FORCE_DECOMPRESS_X1
-
-/* *************************/
-/* double-symbols decoding */
-/* *************************/
-
-typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */
-typedef struct { BYTE symbol; } sortedSymbol_t;
-typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
-typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX];
-
-/**
- * Constructs a HUF_DEltX2 in a U32.
- */
-static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level)
-{
- U32 seq;
- DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0);
- DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2);
- DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3);
- DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32));
- if (MEM_isLittleEndian()) {
- seq = level == 1 ? symbol : (baseSeq + (symbol << 8));
- return seq + (nbBits << 16) + ((U32)level << 24);
- } else {
- seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol);
- return (seq << 16) + (nbBits << 8) + (U32)level;
- }
-}
-
-/**
- * Constructs a HUF_DEltX2.
- */
-static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level)
-{
- HUF_DEltX2 DElt;
- U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
- DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val));
- ZSTD_memcpy(&DElt, &val, sizeof(val));
- return DElt;
-}
-
-/**
- * Constructs 2 HUF_DEltX2s and packs them into a U64.
- */
-static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level)
-{
- U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
- return (U64)DElt + ((U64)DElt << 32);
-}
-
-/**
- * Fills the DTable rank with all the symbols from [begin, end) that are each
- * nbBits long.
- *
- * @param DTableRank The start of the rank in the DTable.
- * @param begin The first symbol to fill (inclusive).
- * @param end The last symbol to fill (exclusive).
- * @param nbBits Each symbol is nbBits long.
- * @param tableLog The table log.
- * @param baseSeq If level == 1 { 0 } else { the first level symbol }
- * @param level The level in the table. Must be 1 or 2.
- */
-static void HUF_fillDTableX2ForWeight(
- HUF_DEltX2* DTableRank,
- sortedSymbol_t const* begin, sortedSymbol_t const* end,
- U32 nbBits, U32 tableLog,
- U16 baseSeq, int const level)
-{
- U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */);
- const sortedSymbol_t* ptr;
- assert(level >= 1 && level <= 2);
- switch (length) {
- case 1:
- for (ptr = begin; ptr != end; ++ptr) {
- HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
- *DTableRank++ = DElt;
- }
- break;
- case 2:
- for (ptr = begin; ptr != end; ++ptr) {
- HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
- DTableRank[0] = DElt;
- DTableRank[1] = DElt;
- DTableRank += 2;
- }
- break;
- case 4:
- for (ptr = begin; ptr != end; ++ptr) {
- U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
- ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
- ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
- DTableRank += 4;
- }
- break;
- case 8:
- for (ptr = begin; ptr != end; ++ptr) {
- U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
- ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
- ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
- ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
- ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
- DTableRank += 8;
- }
- break;
- default:
- for (ptr = begin; ptr != end; ++ptr) {
- U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
- HUF_DEltX2* const DTableRankEnd = DTableRank + length;
- for (; DTableRank != DTableRankEnd; DTableRank += 8) {
- ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
- ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
- ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
- ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
- }
- }
- break;
- }
-}
-
-/* HUF_fillDTableX2Level2() :
- * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
-static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits,
- const U32* rankVal, const int minWeight, const int maxWeight1,
- const sortedSymbol_t* sortedSymbols, U32 const* rankStart,
- U32 nbBitsBaseline, U16 baseSeq)
-{
- /* Fill skipped values (all positions up to rankVal[minWeight]).
- * These are positions only get a single symbol because the combined weight
- * is too large.
- */
- if (minWeight>1) {
- U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */);
- U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1);
- int const skipSize = rankVal[minWeight];
- assert(length > 1);
- assert((U32)skipSize < length);
- switch (length) {
- case 2:
- assert(skipSize == 1);
- ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2));
- break;
- case 4:
- assert(skipSize <= 4);
- ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2));
- ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2));
- break;
- default:
- {
- int i;
- for (i = 0; i < skipSize; i += 8) {
- ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2));
- ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2));
- ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2));
- ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2));
- }
- }
- }
- }
-
- /* Fill each of the second level symbols by weight. */
- {
- int w;
- for (w = minWeight; w < maxWeight1; ++w) {
- int const begin = rankStart[w];
- int const end = rankStart[w+1];
- U32 const nbBits = nbBitsBaseline - w;
- U32 const totalBits = nbBits + consumedBits;
- HUF_fillDTableX2ForWeight(
- DTable + rankVal[w],
- sortedSymbols + begin, sortedSymbols + end,
- totalBits, targetLog,
- baseSeq, /* level */ 2);
- }
- }
-}
-
-static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
- const sortedSymbol_t* sortedList,
- const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
- const U32 nbBitsBaseline)
-{
- U32* const rankVal = rankValOrigin[0];
- const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
- const U32 minBits = nbBitsBaseline - maxWeight;
- int w;
- int const wEnd = (int)maxWeight + 1;
-
- /* Fill DTable in order of weight. */
- for (w = 1; w < wEnd; ++w) {
- int const begin = (int)rankStart[w];
- int const end = (int)rankStart[w+1];
- U32 const nbBits = nbBitsBaseline - w;
-
- if (targetLog-nbBits >= minBits) {
- /* Enough room for a second symbol. */
- int start = rankVal[w];
- U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */);
- int minWeight = nbBits + scaleLog;
- int s;
- if (minWeight < 1) minWeight = 1;
- /* Fill the DTable for every symbol of weight w.
- * These symbols get at least 1 second symbol.
- */
- for (s = begin; s != end; ++s) {
- HUF_fillDTableX2Level2(
- DTable + start, targetLog, nbBits,
- rankValOrigin[nbBits], minWeight, wEnd,
- sortedList, rankStart,
- nbBitsBaseline, sortedList[s].symbol);
- start += length;
- }
- } else {
- /* Only a single symbol. */
- HUF_fillDTableX2ForWeight(
- DTable + rankVal[w],
- sortedList + begin, sortedList + end,
- nbBits, targetLog,
- /* baseSeq */ 0, /* level */ 1);
- }
- }
-}
-
-typedef struct {
- rankValCol_t rankVal[HUF_TABLELOG_MAX];
- U32 rankStats[HUF_TABLELOG_MAX + 1];
- U32 rankStart0[HUF_TABLELOG_MAX + 3];
- sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
- BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
- U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
-} HUF_ReadDTableX2_Workspace;
-
-size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
- const void* src, size_t srcSize,
- void* workSpace, size_t wkspSize)
-{
- return HUF_readDTableX2_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
-size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable,
- const void* src, size_t srcSize,
- void* workSpace, size_t wkspSize, int bmi2)
-{
- U32 tableLog, maxW, nbSymbols;
- DTableDesc dtd = HUF_getDTableDesc(DTable);
- U32 maxTableLog = dtd.maxTableLog;
- size_t iSize;
- void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
- HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
- U32 *rankStart;
-
- HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace;
-
- if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC);
-
- rankStart = wksp->rankStart0 + 1;
- ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats));
- ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0));
-
- DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
- if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
- /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
-
- iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), bmi2);
- if (HUF_isError(iSize)) return iSize;
-
- /* check result */
- if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
- if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG;
-
- /* find maxWeight */
- for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
-
- /* Get start index of each weight */
- { U32 w, nextRankStart = 0;
- for (w=1; w<maxW+1; w++) {
- U32 curr = nextRankStart;
- nextRankStart += wksp->rankStats[w];
- rankStart[w] = curr;
- }
- rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
- rankStart[maxW+1] = nextRankStart;
- }
-
- /* sort symbols by weight */
- { U32 s;
- for (s=0; s<nbSymbols; s++) {
- U32 const w = wksp->weightList[s];
- U32 const r = rankStart[w]++;
- wksp->sortedSymbol[r].symbol = (BYTE)s;
- }
- rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
- }
-
- /* Build rankVal */
- { U32* const rankVal0 = wksp->rankVal[0];
- { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */
- U32 nextRankVal = 0;
- U32 w;
- for (w=1; w<maxW+1; w++) {
- U32 curr = nextRankVal;
- nextRankVal += wksp->rankStats[w] << (w+rescale);
- rankVal0[w] = curr;
- } }
- { U32 const minBits = tableLog+1 - maxW;
- U32 consumed;
- for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
- U32* const rankValPtr = wksp->rankVal[consumed];
- U32 w;
- for (w = 1; w < maxW+1; w++) {
- rankValPtr[w] = rankVal0[w] >> consumed;
- } } } }
-
- HUF_fillDTableX2(dt, maxTableLog,
- wksp->sortedSymbol,
- wksp->rankStart0, wksp->rankVal, maxW,
- tableLog+1);
-
- dtd.tableLog = (BYTE)maxTableLog;
- dtd.tableType = 1;
- ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
- return iSize;
-}
-
-
-FORCE_INLINE_TEMPLATE U32
-HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
-{
- size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
- ZSTD_memcpy(op, &dt[val].sequence, 2);
- BIT_skipBits(DStream, dt[val].nbBits);
- return dt[val].length;
-}
-
-FORCE_INLINE_TEMPLATE U32
-HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
-{
- size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
- ZSTD_memcpy(op, &dt[val].sequence, 1);
- if (dt[val].length==1) {
- BIT_skipBits(DStream, dt[val].nbBits);
- } else {
- if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
- BIT_skipBits(DStream, dt[val].nbBits);
- if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
- /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
- DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8);
- }
- }
- return 1;
-}
-
-#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \
- ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
-
-#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
- if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
- ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
-
-#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
- if (MEM_64bits()) \
- ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
-
-HINT_INLINE size_t
-HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd,
- const HUF_DEltX2* const dt, const U32 dtLog)
-{
- BYTE* const pStart = p;
-
- /* up to 8 symbols at a time */
- if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) {
- if (dtLog <= 11 && MEM_64bits()) {
- /* up to 10 symbols at a time */
- while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) {
- HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
- HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
- HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
- HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
- HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
- }
- } else {
- /* up to 8 symbols at a time */
- while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) {
- HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
- HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
- HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
- HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
- }
- }
- } else {
- BIT_reloadDStream(bitDPtr);
- }
-
- /* closer to end : up to 2 symbols at a time */
- if ((size_t)(pEnd - p) >= 2) {
- while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2))
- HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
-
- while (p <= pEnd-2)
- HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
- }
-
- if (p < pEnd)
- p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog);
-
- return p-pStart;
-}
-
-FORCE_INLINE_TEMPLATE size_t
-HUF_decompress1X2_usingDTable_internal_body(
- void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable)
-{
- BIT_DStream_t bitD;
-
- /* Init */
- CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
-
- /* decode */
- { BYTE* const ostart = (BYTE*) dst;
- BYTE* const oend = ostart + dstSize;
- const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */
- const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
- DTableDesc const dtd = HUF_getDTableDesc(DTable);
- HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog);
- }
-
- /* check */
- if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
-
- /* decoded size */
- return dstSize;
-}
-FORCE_INLINE_TEMPLATE size_t
-HUF_decompress4X2_usingDTable_internal_body(
- void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable)
-{
- if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
-
- { const BYTE* const istart = (const BYTE*) cSrc;
- BYTE* const ostart = (BYTE*) dst;
- BYTE* const oend = ostart + dstSize;
- BYTE* const olimit = oend - (sizeof(size_t)-1);
- const void* const dtPtr = DTable+1;
- const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
-
- /* Init */
- BIT_DStream_t bitD1;
- BIT_DStream_t bitD2;
- BIT_DStream_t bitD3;
- BIT_DStream_t bitD4;
- size_t const length1 = MEM_readLE16(istart);
- size_t const length2 = MEM_readLE16(istart+2);
- size_t const length3 = MEM_readLE16(istart+4);
- size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
- const BYTE* const istart1 = istart + 6; /* jumpTable */
- const BYTE* const istart2 = istart1 + length1;
- const BYTE* const istart3 = istart2 + length2;
- const BYTE* const istart4 = istart3 + length3;
- size_t const segmentSize = (dstSize+3) / 4;
- BYTE* const opStart2 = ostart + segmentSize;
- BYTE* const opStart3 = opStart2 + segmentSize;
- BYTE* const opStart4 = opStart3 + segmentSize;
- BYTE* op1 = ostart;
- BYTE* op2 = opStart2;
- BYTE* op3 = opStart3;
- BYTE* op4 = opStart4;
- U32 endSignal = 1;
- DTableDesc const dtd = HUF_getDTableDesc(DTable);
- U32 const dtLog = dtd.tableLog;
-
- if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
- if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */
- CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
- CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
- CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
- CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
-
- /* 16-32 symbols per loop (4-8 symbols per stream) */
- if ((size_t)(oend - op4) >= sizeof(size_t)) {
- for ( ; (endSignal) & (op4 < olimit); ) {
-#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
- HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
- HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
- HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
- HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
- HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
- HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
- HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
- HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
- endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
- endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
- HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
- HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
- HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
- HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
- HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
- HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
- HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
- HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
- endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
- endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
-#else
- HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
- HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
- HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
- HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
- HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
- HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
- HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
- HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
- HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
- HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
- HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
- HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
- HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
- HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
- HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
- HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
- endSignal = (U32)LIKELY((U32)
- (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished)
- & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished)
- & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished)
- & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished));
-#endif
- }
- }
-
- /* check corruption */
- if (op1 > opStart2) return ERROR(corruption_detected);
- if (op2 > opStart3) return ERROR(corruption_detected);
- if (op3 > opStart4) return ERROR(corruption_detected);
- /* note : op4 already verified within main loop */
-
- /* finish bitStreams one by one */
- HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
- HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
- HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
- HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
-
- /* check */
- { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
- if (!endCheck) return ERROR(corruption_detected); }
-
- /* decoded size */
- return dstSize;
- }
-}
-
-#if HUF_NEED_BMI2_FUNCTION
-static BMI2_TARGET_ATTRIBUTE
-size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
- size_t cSrcSize, HUF_DTable const* DTable) {
- return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
-}
-#endif
-
-#if HUF_NEED_DEFAULT_FUNCTION
-static
-size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
- size_t cSrcSize, HUF_DTable const* DTable) {
- return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
-}
-#endif
-
-#if ZSTD_ENABLE_ASM_X86_64_BMI2
-
-HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN;
-
-static HUF_ASM_X86_64_BMI2_ATTRS size_t
-HUF_decompress4X2_usingDTable_internal_bmi2_asm(
- void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable) {
- void const* dt = DTable + 1;
- const BYTE* const iend = (const BYTE*)cSrc + 6;
- BYTE* const oend = (BYTE*)dst + dstSize;
- HUF_DecompressAsmArgs args;
- {
- size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
- FORWARD_IF_ERROR(ret, "Failed to init asm args");
- if (ret != 0)
- return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
- }
-
- assert(args.ip[0] >= args.ilimit);
- HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(&args);
-
- /* note : op4 already verified within main loop */
- assert(args.ip[0] >= iend);
- assert(args.ip[1] >= iend);
- assert(args.ip[2] >= iend);
- assert(args.ip[3] >= iend);
- assert(args.op[3] <= oend);
- (void)iend;
-
- /* finish bitStreams one by one */
- {
- size_t const segmentSize = (dstSize+3) / 4;
- BYTE* segmentEnd = (BYTE*)dst;
- int i;
- for (i = 0; i < 4; ++i) {
- BIT_DStream_t bit;
- if (segmentSize <= (size_t)(oend - segmentEnd))
- segmentEnd += segmentSize;
- else
- segmentEnd = oend;
- FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
- args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG);
- if (args.op[i] != segmentEnd)
- return ERROR(corruption_detected);
- }
- }
-
- /* decoded size */
- return dstSize;
-}
-#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
-
-static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
- size_t cSrcSize, HUF_DTable const* DTable, int bmi2)
-{
-#if DYNAMIC_BMI2
- if (bmi2) {
-# if ZSTD_ENABLE_ASM_X86_64_BMI2
- return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-# else
- return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
-# endif
- }
-#else
- (void)bmi2;
-#endif
-
-#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
- return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-#else
- return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
-#endif
-}
-
-HUF_DGEN(HUF_decompress1X2_usingDTable_internal)
-
-size_t HUF_decompress1X2_usingDTable(
- void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable)
-{
- DTableDesc dtd = HUF_getDTableDesc(DTable);
- if (dtd.tableType != 1) return ERROR(GENERIC);
- return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
-size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- void* workSpace, size_t wkspSize)
-{
- const BYTE* ip = (const BYTE*) cSrc;
-
- size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize,
- workSpace, wkspSize);
- if (HUF_isError(hSize)) return hSize;
- if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
- ip += hSize; cSrcSize -= hSize;
-
- return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0);
-}
-
-
-size_t HUF_decompress4X2_usingDTable(
- void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable)
-{
- DTableDesc dtd = HUF_getDTableDesc(DTable);
- if (dtd.tableType != 1) return ERROR(GENERIC);
- return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
-static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- void* workSpace, size_t wkspSize, int bmi2)
-{
- const BYTE* ip = (const BYTE*) cSrc;
-
- size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize,
- workSpace, wkspSize);
- if (HUF_isError(hSize)) return hSize;
- if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
- ip += hSize; cSrcSize -= hSize;
-
- return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
-}
-
-size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- void* workSpace, size_t wkspSize)
-{
- return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
-
-#endif /* HUF_FORCE_DECOMPRESS_X1 */
-
-
-/* ***********************************/
-/* Universal decompression selectors */
-/* ***********************************/
-
-size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable)
-{
- DTableDesc const dtd = HUF_getDTableDesc(DTable);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
- (void)dtd;
- assert(dtd.tableType == 0);
- return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
- (void)dtd;
- assert(dtd.tableType == 1);
- return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#else
- return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
- HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#endif
-}
-
-size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
- const void* cSrc, size_t cSrcSize,
- const HUF_DTable* DTable)
-{
- DTableDesc const dtd = HUF_getDTableDesc(DTable);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
- (void)dtd;
- assert(dtd.tableType == 0);
- return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
- (void)dtd;
- assert(dtd.tableType == 1);
- return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#else
- return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
- HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#endif
-}
-
-
-#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
-typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
-static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] =
-{
- /* single, double, quad */
- {{0,0}, {1,1}}, /* Q==0 : impossible */
- {{0,0}, {1,1}}, /* Q==1 : impossible */
- {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */
- {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */
- {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */
- {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */
- {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */
- {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */
- {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */
- {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */
- {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */
- {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */
- {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */
- {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */
- {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */
- {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */
-};
-#endif
-
-/** HUF_selectDecoder() :
- * Tells which decoder is likely to decode faster,
- * based on a set of pre-computed metrics.
- * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
- * Assumption : 0 < dstSize <= 128 KB */
-U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
-{
- assert(dstSize > 0);
- assert(dstSize <= 128*1024);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
- (void)dstSize;
- (void)cSrcSize;
- return 0;
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
- (void)dstSize;
- (void)cSrcSize;
- return 1;
-#else
- /* decoder timing evaluation */
- { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */
- U32 const D256 = (U32)(dstSize >> 8);
- U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
- U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
- DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */
- return DTime1 < DTime0;
- }
-#endif
-}
-
-
-size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
- size_t dstSize, const void* cSrc,
- size_t cSrcSize, void* workSpace,
- size_t wkspSize)
-{
- /* validation checks */
- if (dstSize == 0) return ERROR(dstSize_tooSmall);
- if (cSrcSize == 0) return ERROR(corruption_detected);
-
- { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
- (void)algoNb;
- assert(algoNb == 0);
- return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
- (void)algoNb;
- assert(algoNb == 1);
- return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#else
- return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
- cSrcSize, workSpace, wkspSize):
- HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#endif
- }
-}
-
-size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize,
- void* workSpace, size_t wkspSize)
-{
- /* validation checks */
- if (dstSize == 0) return ERROR(dstSize_tooSmall);
- if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
- if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
- if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
-
- { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
- (void)algoNb;
- assert(algoNb == 0);
- return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
- cSrcSize, workSpace, wkspSize);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
- (void)algoNb;
- assert(algoNb == 1);
- return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
- cSrcSize, workSpace, wkspSize);
-#else
- return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
- cSrcSize, workSpace, wkspSize):
- HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
- cSrcSize, workSpace, wkspSize);
-#endif
- }
-}
-
-
-size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
-{
- DTableDesc const dtd = HUF_getDTableDesc(DTable);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
- (void)dtd;
- assert(dtd.tableType == 0);
- return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
- (void)dtd;
- assert(dtd.tableType == 1);
- return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
-#else
- return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
- HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
-#endif
-}
-
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
-{
- const BYTE* ip = (const BYTE*) cSrc;
-
- size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
- if (HUF_isError(hSize)) return hSize;
- if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
- ip += hSize; cSrcSize -= hSize;
-
- return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
-}
-#endif
-
-size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
-{
- DTableDesc const dtd = HUF_getDTableDesc(DTable);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
- (void)dtd;
- assert(dtd.tableType == 0);
- return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
- (void)dtd;
- assert(dtd.tableType == 1);
- return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
-#else
- return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
- HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
-#endif
-}
-
-size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
-{
- /* validation checks */
- if (dstSize == 0) return ERROR(dstSize_tooSmall);
- if (cSrcSize == 0) return ERROR(corruption_detected);
-
- { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
- (void)algoNb;
- assert(algoNb == 0);
- return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
- (void)algoNb;
- assert(algoNb == 1);
- return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
-#else
- return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) :
- HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
-#endif
- }
-}
-
-#ifndef ZSTD_NO_UNUSED_FUNCTIONS
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize)
-{
- U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
- return HUF_readDTableX1_wksp(DTable, src, srcSize,
- workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize)
-{
- U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
- return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
- workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
- HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX);
- return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize)
-{
- U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
- return HUF_readDTableX2_wksp(DTable, src, srcSize,
- workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize)
-{
- U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
- return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
- workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
- HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
- return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
- U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
- return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
- workSpace, sizeof(workSpace));
-}
-size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
- HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX);
- return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize)
-{
- U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
- return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
- workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
- HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
- return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
-}
-#endif
-
-typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
-
-size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
-#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
- static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 };
-#endif
-
- /* validation checks */
- if (dstSize == 0) return ERROR(dstSize_tooSmall);
- if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
- if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
- if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
-
- { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
- (void)algoNb;
- assert(algoNb == 0);
- return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
- (void)algoNb;
- assert(algoNb == 1);
- return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize);
-#else
- return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
-#endif
- }
-}
-
-size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
- /* validation checks */
- if (dstSize == 0) return ERROR(dstSize_tooSmall);
- if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
- if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
- if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
-
- { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
- (void)algoNb;
- assert(algoNb == 0);
- return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
- (void)algoNb;
- assert(algoNb == 1);
- return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
-#else
- return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
- HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
-#endif
- }
-}
-
-size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
-{
- U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
- return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
- workSpace, sizeof(workSpace));
-}
-
-size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
- const void* cSrc, size_t cSrcSize)
-{
- U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
- return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
- workSpace, sizeof(workSpace));
-}
-#endif
stage1/zstd/lib/decompress/zstd_ddict.c
@@ -1,244 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-/* zstd_ddict.c :
- * concentrates all logic that needs to know the internals of ZSTD_DDict object */
-
-/*-*******************************************************
-* Dependencies
-*********************************************************/
-#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
-#include "../common/cpu.h" /* bmi2 */
-#include "../common/mem.h" /* low level memory routines */
-#define FSE_STATIC_LINKING_ONLY
-#include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
-#include "../common/huf.h"
-#include "zstd_decompress_internal.h"
-#include "zstd_ddict.h"
-
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
-# include "../legacy/zstd_legacy.h"
-#endif
-
-
-
-/*-*******************************************************
-* Types
-*********************************************************/
-struct ZSTD_DDict_s {
- void* dictBuffer;
- const void* dictContent;
- size_t dictSize;
- ZSTD_entropyDTables_t entropy;
- U32 dictID;
- U32 entropyPresent;
- ZSTD_customMem cMem;
-}; /* typedef'd to ZSTD_DDict within "zstd.h" */
-
-const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
-{
- assert(ddict != NULL);
- return ddict->dictContent;
-}
-
-size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
-{
- assert(ddict != NULL);
- return ddict->dictSize;
-}
-
-void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
-{
- DEBUGLOG(4, "ZSTD_copyDDictParameters");
- assert(dctx != NULL);
- assert(ddict != NULL);
- dctx->dictID = ddict->dictID;
- dctx->prefixStart = ddict->dictContent;
- dctx->virtualStart = ddict->dictContent;
- dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
- dctx->previousDstEnd = dctx->dictEnd;
-#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
- dctx->dictContentBeginForFuzzing = dctx->prefixStart;
- dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
-#endif
- if (ddict->entropyPresent) {
- dctx->litEntropy = 1;
- dctx->fseEntropy = 1;
- dctx->LLTptr = ddict->entropy.LLTable;
- dctx->MLTptr = ddict->entropy.MLTable;
- dctx->OFTptr = ddict->entropy.OFTable;
- dctx->HUFptr = ddict->entropy.hufTable;
- dctx->entropy.rep[0] = ddict->entropy.rep[0];
- dctx->entropy.rep[1] = ddict->entropy.rep[1];
- dctx->entropy.rep[2] = ddict->entropy.rep[2];
- } else {
- dctx->litEntropy = 0;
- dctx->fseEntropy = 0;
- }
-}
-
-
-static size_t
-ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
- ZSTD_dictContentType_e dictContentType)
-{
- ddict->dictID = 0;
- ddict->entropyPresent = 0;
- if (dictContentType == ZSTD_dct_rawContent) return 0;
-
- if (ddict->dictSize < 8) {
- if (dictContentType == ZSTD_dct_fullDict)
- return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
- return 0; /* pure content mode */
- }
- { U32 const magic = MEM_readLE32(ddict->dictContent);
- if (magic != ZSTD_MAGIC_DICTIONARY) {
- if (dictContentType == ZSTD_dct_fullDict)
- return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
- return 0; /* pure content mode */
- }
- }
- ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
-
- /* load entropy tables */
- RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
- &ddict->entropy, ddict->dictContent, ddict->dictSize)),
- dictionary_corrupted, "");
- ddict->entropyPresent = 1;
- return 0;
-}
-
-
-static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
- const void* dict, size_t dictSize,
- ZSTD_dictLoadMethod_e dictLoadMethod,
- ZSTD_dictContentType_e dictContentType)
-{
- if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
- ddict->dictBuffer = NULL;
- ddict->dictContent = dict;
- if (!dict) dictSize = 0;
- } else {
- void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem);
- ddict->dictBuffer = internalBuffer;
- ddict->dictContent = internalBuffer;
- if (!internalBuffer) return ERROR(memory_allocation);
- ZSTD_memcpy(internalBuffer, dict, dictSize);
- }
- ddict->dictSize = dictSize;
- ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
-
- /* parse dictionary content */
- FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
-
- return 0;
-}
-
-ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
- ZSTD_dictLoadMethod_e dictLoadMethod,
- ZSTD_dictContentType_e dictContentType,
- ZSTD_customMem customMem)
-{
- if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
-
- { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem);
- if (ddict == NULL) return NULL;
- ddict->cMem = customMem;
- { size_t const initResult = ZSTD_initDDict_internal(ddict,
- dict, dictSize,
- dictLoadMethod, dictContentType);
- if (ZSTD_isError(initResult)) {
- ZSTD_freeDDict(ddict);
- return NULL;
- } }
- return ddict;
- }
-}
-
-/*! ZSTD_createDDict() :
-* Create a digested dictionary, to start decompression without startup delay.
-* `dict` content is copied inside DDict.
-* Consequently, `dict` can be released after `ZSTD_DDict` creation */
-ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
-{
- ZSTD_customMem const allocator = { NULL, NULL, NULL };
- return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
-}
-
-/*! ZSTD_createDDict_byReference() :
- * Create a digested dictionary, to start decompression without startup delay.
- * Dictionary content is simply referenced, it will be accessed during decompression.
- * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
-ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
-{
- ZSTD_customMem const allocator = { NULL, NULL, NULL };
- return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
-}
-
-
-const ZSTD_DDict* ZSTD_initStaticDDict(
- void* sBuffer, size_t sBufferSize,
- const void* dict, size_t dictSize,
- ZSTD_dictLoadMethod_e dictLoadMethod,
- ZSTD_dictContentType_e dictContentType)
-{
- size_t const neededSpace = sizeof(ZSTD_DDict)
- + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
- ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
- assert(sBuffer != NULL);
- assert(dict != NULL);
- if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
- if (sBufferSize < neededSpace) return NULL;
- if (dictLoadMethod == ZSTD_dlm_byCopy) {
- ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */
- dict = ddict+1;
- }
- if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
- dict, dictSize,
- ZSTD_dlm_byRef, dictContentType) ))
- return NULL;
- return ddict;
-}
-
-
-size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
-{
- if (ddict==NULL) return 0; /* support free on NULL */
- { ZSTD_customMem const cMem = ddict->cMem;
- ZSTD_customFree(ddict->dictBuffer, cMem);
- ZSTD_customFree(ddict, cMem);
- return 0;
- }
-}
-
-/*! ZSTD_estimateDDictSize() :
- * Estimate amount of memory that will be needed to create a dictionary for decompression.
- * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
-size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
-{
- return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
-}
-
-size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
-{
- if (ddict==NULL) return 0; /* support sizeof on NULL */
- return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
-}
-
-/*! ZSTD_getDictID_fromDDict() :
- * Provides the dictID of the dictionary loaded into `ddict`.
- * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
- * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
-unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
-{
- if (ddict==NULL) return 0;
- return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
-}
stage1/zstd/lib/decompress/zstd_ddict.h
@@ -1,44 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-
-#ifndef ZSTD_DDICT_H
-#define ZSTD_DDICT_H
-
-/*-*******************************************************
- * Dependencies
- *********************************************************/
-#include "../common/zstd_deps.h" /* size_t */
-#include "../zstd.h" /* ZSTD_DDict, and several public functions */
-
-
-/*-*******************************************************
- * Interface
- *********************************************************/
-
-/* note: several prototypes are already published in `zstd.h` :
- * ZSTD_createDDict()
- * ZSTD_createDDict_byReference()
- * ZSTD_createDDict_advanced()
- * ZSTD_freeDDict()
- * ZSTD_initStaticDDict()
- * ZSTD_sizeof_DDict()
- * ZSTD_estimateDDictSize()
- * ZSTD_getDictID_fromDict()
- */
-
-const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict);
-size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict);
-
-void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
-
-
-
-#endif /* ZSTD_DDICT_H */
stage1/zstd/lib/decompress/zstd_decompress.c
@@ -1,2230 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-
-/* ***************************************************************
-* Tuning parameters
-*****************************************************************/
-/*!
- * HEAPMODE :
- * Select how default decompression function ZSTD_decompress() allocates its context,
- * on stack (0), or into heap (1, default; requires malloc()).
- * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected.
- */
-#ifndef ZSTD_HEAPMODE
-# define ZSTD_HEAPMODE 1
-#endif
-
-/*!
-* LEGACY_SUPPORT :
-* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+)
-*/
-#ifndef ZSTD_LEGACY_SUPPORT
-# define ZSTD_LEGACY_SUPPORT 0
-#endif
-
-/*!
- * MAXWINDOWSIZE_DEFAULT :
- * maximum window size accepted by DStream __by default__.
- * Frames requiring more memory will be rejected.
- * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize().
- */
-#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
-# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1)
-#endif
-
-/*!
- * NO_FORWARD_PROGRESS_MAX :
- * maximum allowed nb of calls to ZSTD_decompressStream()
- * without any forward progress
- * (defined as: no byte read from input, and no byte flushed to output)
- * before triggering an error.
- */
-#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX
-# define ZSTD_NO_FORWARD_PROGRESS_MAX 16
-#endif
-
-
-/*-*******************************************************
-* Dependencies
-*********************************************************/
-#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
-#include "../common/mem.h" /* low level memory routines */
-#define FSE_STATIC_LINKING_ONLY
-#include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
-#include "../common/huf.h"
-#include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
-#include "../common/zstd_internal.h" /* blockProperties_t */
-#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
-#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
-#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */
-
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
-# include "../legacy/zstd_legacy.h"
-#endif
-
-
-
-/*************************************
- * Multiple DDicts Hashset internals *
- *************************************/
-
-#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4
-#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
- * Currently, that means a 0.75 load factor.
- * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
- * the load factor of the ddict hash set.
- */
-
-#define DDICT_HASHSET_TABLE_BASE_SIZE 64
-#define DDICT_HASHSET_RESIZE_FACTOR 2
-
-/* Hash function to determine starting position of dict insertion within the table
- * Returns an index between [0, hashSet->ddictPtrTableSize]
- */
-static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) {
- const U64 hash = XXH64(&dictID, sizeof(U32), 0);
- /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */
- return hash & (hashSet->ddictPtrTableSize - 1);
-}
-
-/* Adds DDict to a hashset without resizing it.
- * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set.
- * Returns 0 if successful, or a zstd error code if something went wrong.
- */
-static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) {
- const U32 dictID = ZSTD_getDictID_fromDDict(ddict);
- size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
- const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
- RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
- DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
- while (hashSet->ddictPtrTable[idx] != NULL) {
- /* Replace existing ddict if inserting ddict with same dictID */
- if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) {
- DEBUGLOG(4, "DictID already exists, replacing rather than adding");
- hashSet->ddictPtrTable[idx] = ddict;
- return 0;
- }
- idx &= idxRangeMask;
- idx++;
- }
- DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
- hashSet->ddictPtrTable[idx] = ddict;
- hashSet->ddictPtrCount++;
- return 0;
-}
-
-/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and
- * rehashes all values, allocates new table, frees old table.
- * Returns 0 on success, otherwise a zstd error code.
- */
-static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
- size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR;
- const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem);
- const ZSTD_DDict** oldTable = hashSet->ddictPtrTable;
- size_t oldTableSize = hashSet->ddictPtrTableSize;
- size_t i;
-
- DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize);
- RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
- hashSet->ddictPtrTable = newTable;
- hashSet->ddictPtrTableSize = newTableSize;
- hashSet->ddictPtrCount = 0;
- for (i = 0; i < oldTableSize; ++i) {
- if (oldTable[i] != NULL) {
- FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), "");
- }
- }
- ZSTD_customFree((void*)oldTable, customMem);
- DEBUGLOG(4, "Finished re-hash");
- return 0;
-}
-
-/* Fetches a DDict with the given dictID
- * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL.
- */
-static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) {
- size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
- const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
- DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
- for (;;) {
- size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
- if (currDictID == dictID || currDictID == 0) {
- /* currDictID == 0 implies a NULL ddict entry */
- break;
- } else {
- idx &= idxRangeMask; /* Goes to start of table when we reach the end */
- idx++;
- }
- }
- DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
- return hashSet->ddictPtrTable[idx];
-}
-
-/* Allocates space for and returns a ddict hash set
- * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with.
- * Returns NULL if allocation failed.
- */
-static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
- ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
- DEBUGLOG(4, "Allocating new hash set");
- if (!ret)
- return NULL;
- ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem);
- if (!ret->ddictPtrTable) {
- ZSTD_customFree(ret, customMem);
- return NULL;
- }
- ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
- ret->ddictPtrCount = 0;
- return ret;
-}
-
-/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself.
- * Note: The ZSTD_DDict* within the table are NOT freed.
- */
-static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
- DEBUGLOG(4, "Freeing ddict hash set");
- if (hashSet && hashSet->ddictPtrTable) {
- ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem);
- }
- if (hashSet) {
- ZSTD_customFree(hashSet, customMem);
- }
-}
-
-/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
- * Returns 0 on success, or a ZSTD error.
- */
-static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) {
- DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
- if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) {
- FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
- }
- FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), "");
- return 0;
-}
-
-/*-*************************************************************
-* Context management
-***************************************************************/
-size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
-{
- if (dctx==NULL) return 0; /* support sizeof NULL */
- return sizeof(*dctx)
- + ZSTD_sizeof_DDict(dctx->ddictLocal)
- + dctx->inBuffSize + dctx->outBuffSize;
-}
-
-size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
-
-
-static size_t ZSTD_startingInputLength(ZSTD_format_e format)
-{
- size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format);
- /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
- assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) );
- return startingInputLength;
-}
-
-static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
-{
- assert(dctx->streamStage == zdss_init);
- dctx->format = ZSTD_f_zstd1;
- dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
- dctx->outBufferMode = ZSTD_bm_buffered;
- dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
- dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
-}
-
-static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
-{
- dctx->staticSize = 0;
- dctx->ddict = NULL;
- dctx->ddictLocal = NULL;
- dctx->dictEnd = NULL;
- dctx->ddictIsCold = 0;
- dctx->dictUses = ZSTD_dont_use;
- dctx->inBuff = NULL;
- dctx->inBuffSize = 0;
- dctx->outBuffSize = 0;
- dctx->streamStage = zdss_init;
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
- dctx->legacyContext = NULL;
- dctx->previousLegacyVersion = 0;
-#endif
- dctx->noForwardProgress = 0;
- dctx->oversizedDuration = 0;
-#if DYNAMIC_BMI2
- dctx->bmi2 = ZSTD_cpuSupportsBmi2();
-#endif
- dctx->ddictSet = NULL;
- ZSTD_DCtx_resetParameters(dctx);
-#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
- dctx->dictContentEndForFuzzing = NULL;
-#endif
-}
-
-ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
-{
- ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace;
-
- if ((size_t)workspace & 7) return NULL; /* 8-aligned */
- if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */
-
- ZSTD_initDCtx_internal(dctx);
- dctx->staticSize = workspaceSize;
- dctx->inBuff = (char*)(dctx+1);
- return dctx;
-}
-
-static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) {
- if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
-
- { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem);
- if (!dctx) return NULL;
- dctx->customMem = customMem;
- ZSTD_initDCtx_internal(dctx);
- return dctx;
- }
-}
-
-ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
-{
- return ZSTD_createDCtx_internal(customMem);
-}
-
-ZSTD_DCtx* ZSTD_createDCtx(void)
-{
- DEBUGLOG(3, "ZSTD_createDCtx");
- return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
-}
-
-static void ZSTD_clearDict(ZSTD_DCtx* dctx)
-{
- ZSTD_freeDDict(dctx->ddictLocal);
- dctx->ddictLocal = NULL;
- dctx->ddict = NULL;
- dctx->dictUses = ZSTD_dont_use;
-}
-
-size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
-{
- if (dctx==NULL) return 0; /* support free on NULL */
- RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");
- { ZSTD_customMem const cMem = dctx->customMem;
- ZSTD_clearDict(dctx);
- ZSTD_customFree(dctx->inBuff, cMem);
- dctx->inBuff = NULL;
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
- if (dctx->legacyContext)
- ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
-#endif
- if (dctx->ddictSet) {
- ZSTD_freeDDictHashSet(dctx->ddictSet, cMem);
- dctx->ddictSet = NULL;
- }
- ZSTD_customFree(dctx, cMem);
- return 0;
- }
-}
-
-/* no longer useful */
-void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
-{
- size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);
- ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */
-}
-
-/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on
- * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then
- * accordingly sets the ddict to be used to decompress the frame.
- *
- * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is.
- *
- * ZSTD_d_refMultipleDDicts must be enabled for this function to be called.
- */
-static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) {
- assert(dctx->refMultipleDDicts && dctx->ddictSet);
- DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame");
- if (dctx->ddict) {
- const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
- if (frameDDict) {
- DEBUGLOG(4, "DDict found!");
- ZSTD_clearDict(dctx);
- dctx->dictID = dctx->fParams.dictID;
- dctx->ddict = frameDDict;
- dctx->dictUses = ZSTD_use_indefinitely;
- }
- }
-}
-
-
-/*-*************************************************************
- * Frame header decoding
- ***************************************************************/
-
-/*! ZSTD_isFrame() :
- * Tells if the content of `buffer` starts with a valid Frame Identifier.
- * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
- * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
- * Note 3 : Skippable Frame Identifiers are considered valid. */
-unsigned ZSTD_isFrame(const void* buffer, size_t size)
-{
- if (size < ZSTD_FRAMEIDSIZE) return 0;
- { U32 const magic = MEM_readLE32(buffer);
- if (magic == ZSTD_MAGICNUMBER) return 1;
- if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
- }
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
- if (ZSTD_isLegacy(buffer, size)) return 1;
-#endif
- return 0;
-}
-
-/*! ZSTD_isSkippableFrame() :
- * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
- * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
- */
-unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size)
-{
- if (size < ZSTD_FRAMEIDSIZE) return 0;
- { U32 const magic = MEM_readLE32(buffer);
- if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
- }
- return 0;
-}
-
-/** ZSTD_frameHeaderSize_internal() :
- * srcSize must be large enough to reach header size fields.
- * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless.
- * @return : size of the Frame Header
- * or an error code, which can be tested with ZSTD_isError() */
-static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format)
-{
- size_t const minInputSize = ZSTD_startingInputLength(format);
- RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, "");
-
- { BYTE const fhd = ((const BYTE*)src)[minInputSize-1];
- U32 const dictID= fhd & 3;
- U32 const singleSegment = (fhd >> 5) & 1;
- U32 const fcsId = fhd >> 6;
- return minInputSize + !singleSegment
- + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
- + (singleSegment && !fcsId);
- }
-}
-
-/** ZSTD_frameHeaderSize() :
- * srcSize must be >= ZSTD_frameHeaderSize_prefix.
- * @return : size of the Frame Header,
- * or an error code (if srcSize is too small) */
-size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
-{
- return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1);
-}
-
-
-/** ZSTD_getFrameHeader_advanced() :
- * decode Frame Header, or require larger `srcSize`.
- * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless
- * @return : 0, `zfhPtr` is correctly filled,
- * >0, `srcSize` is too small, value is wanted `srcSize` amount,
- * or an error code, which can be tested using ZSTD_isError() */
-size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
-{
- const BYTE* ip = (const BYTE*)src;
- size_t const minInputSize = ZSTD_startingInputLength(format);
-
- ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
- if (srcSize < minInputSize) return minInputSize;
- RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter");
-
- if ( (format != ZSTD_f_zstd1_magicless)
- && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
- if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
- /* skippable frame */
- if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
- return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
- ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));
- zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
- zfhPtr->frameType = ZSTD_skippableFrame;
- return 0;
- }
- RETURN_ERROR(prefix_unknown, "");
- }
-
- /* ensure there is enough `srcSize` to fully read/decode frame header */
- { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format);
- if (srcSize < fhsize) return fhsize;
- zfhPtr->headerSize = (U32)fhsize;
- }
-
- { BYTE const fhdByte = ip[minInputSize-1];
- size_t pos = minInputSize;
- U32 const dictIDSizeCode = fhdByte&3;
- U32 const checksumFlag = (fhdByte>>2)&1;
- U32 const singleSegment = (fhdByte>>5)&1;
- U32 const fcsID = fhdByte>>6;
- U64 windowSize = 0;
- U32 dictID = 0;
- U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
- RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported,
- "reserved bits, must be zero");
-
- if (!singleSegment) {
- BYTE const wlByte = ip[pos++];
- U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
- RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, "");
- windowSize = (1ULL << windowLog);
- windowSize += (windowSize >> 3) * (wlByte&7);
- }
- switch(dictIDSizeCode)
- {
- default:
- assert(0); /* impossible */
- ZSTD_FALLTHROUGH;
- case 0 : break;
- case 1 : dictID = ip[pos]; pos++; break;
- case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
- case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
- }
- switch(fcsID)
- {
- default:
- assert(0); /* impossible */
- ZSTD_FALLTHROUGH;
- case 0 : if (singleSegment) frameContentSize = ip[pos]; break;
- case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
- case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
- case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
- }
- if (singleSegment) windowSize = frameContentSize;
-
- zfhPtr->frameType = ZSTD_frame;
- zfhPtr->frameContentSize = frameContentSize;
- zfhPtr->windowSize = windowSize;
- zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
- zfhPtr->dictID = dictID;
- zfhPtr->checksumFlag = checksumFlag;
- }
- return 0;
-}
-
-/** ZSTD_getFrameHeader() :
- * decode Frame Header, or require larger `srcSize`.
- * note : this function does not consume input, it only reads it.
- * @return : 0, `zfhPtr` is correctly filled,
- * >0, `srcSize` is too small, value is wanted `srcSize` amount,
- * or an error code, which can be tested using ZSTD_isError() */
-size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
-{
- return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);
-}
-
-/** ZSTD_getFrameContentSize() :
- * compatible with legacy mode
- * @return : decompressed size of the single frame pointed to be `src` if known, otherwise
- * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
- * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
-unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
-{
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
- if (ZSTD_isLegacy(src, srcSize)) {
- unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize);
- return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
- }
-#endif
- { ZSTD_frameHeader zfh;
- if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0)
- return ZSTD_CONTENTSIZE_ERROR;
- if (zfh.frameType == ZSTD_skippableFrame) {
- return 0;
- } else {
- return zfh.frameContentSize;
- } }
-}
-
-static size_t readSkippableFrameSize(void const* src, size_t srcSize)
-{
- size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE;
- U32 sizeU32;
-
- RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, "");
-
- sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE);
- RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32,
- frameParameter_unsupported, "");
- {
- size_t const skippableSize = skippableHeaderSize + sizeU32;
- RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, "");
- return skippableSize;
- }
-}
-
-/*! ZSTD_readSkippableFrame() :
- * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
- *
- * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
- * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
- * in the magicVariant.
- *
- * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
- *
- * @return : number of bytes written or a ZSTD error.
- */
-ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
- const void* src, size_t srcSize)
-{
- U32 const magicNumber = MEM_readLE32(src);
- size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
- size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
-
- /* check input validity */
- RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
- RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
- RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
-
- /* deliver payload */
- if (skippableContentSize > 0 && dst != NULL)
- ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
- if (magicVariant != NULL)
- *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
- return skippableContentSize;
-}
-
-/** ZSTD_findDecompressedSize() :
- * compatible with legacy mode
- * `srcSize` must be the exact length of some number of ZSTD compressed and/or
- * skippable frames
- * @return : decompressed size of the frames contained */
-unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
-{
- unsigned long long totalDstSize = 0;
-
- while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) {
- U32 const magicNumber = MEM_readLE32(src);
-
- if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
- size_t const skippableSize = readSkippableFrameSize(src, srcSize);
- if (ZSTD_isError(skippableSize)) {
- return ZSTD_CONTENTSIZE_ERROR;
- }
- assert(skippableSize <= srcSize);
-
- src = (const BYTE *)src + skippableSize;
- srcSize -= skippableSize;
- continue;
- }
-
- { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
- if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
-
- /* check for overflow */
- if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
- totalDstSize += ret;
- }
- { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
- if (ZSTD_isError(frameSrcSize)) {
- return ZSTD_CONTENTSIZE_ERROR;
- }
-
- src = (const BYTE *)src + frameSrcSize;
- srcSize -= frameSrcSize;
- }
- } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
-
- if (srcSize) return ZSTD_CONTENTSIZE_ERROR;
-
- return totalDstSize;
-}
-
-/** ZSTD_getDecompressedSize() :
- * compatible with legacy mode
- * @return : decompressed size if known, 0 otherwise
- note : 0 can mean any of the following :
- - frame content is empty
- - decompressed size field is not present in frame header
- - frame header unknown / not supported
- - frame header not complete (`srcSize` too small) */
-unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
-{
- unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
- ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN);
- return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret;
-}
-
-
-/** ZSTD_decodeFrameHeader() :
- * `headerSize` must be the size provided by ZSTD_frameHeaderSize().
- * If multiple DDict references are enabled, also will choose the correct DDict to use.
- * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
-static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
-{
- size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);
- if (ZSTD_isError(result)) return result; /* invalid header */
- RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
-
- /* Reference DDict requested by frame if dctx references multiple ddicts */
- if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) {
- ZSTD_DCtx_selectFrameDDict(dctx);
- }
-
-#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
- /* Skip the dictID check in fuzzing mode, because it makes the search
- * harder.
- */
- RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),
- dictionary_wrong, "");
-#endif
- dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0;
- if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0);
- dctx->processedCSize += headerSize;
- return 0;
-}
-
-static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret)
-{
- ZSTD_frameSizeInfo frameSizeInfo;
- frameSizeInfo.compressedSize = ret;
- frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
- return frameSizeInfo;
-}
-
-static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize)
-{
- ZSTD_frameSizeInfo frameSizeInfo;
- ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
-
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
- if (ZSTD_isLegacy(src, srcSize))
- return ZSTD_findFrameSizeInfoLegacy(src, srcSize);
-#endif
-
- if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
- && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
- frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize);
- assert(ZSTD_isError(frameSizeInfo.compressedSize) ||
- frameSizeInfo.compressedSize <= srcSize);
- return frameSizeInfo;
- } else {
- const BYTE* ip = (const BYTE*)src;
- const BYTE* const ipstart = ip;
- size_t remainingSize = srcSize;
- size_t nbBlocks = 0;
- ZSTD_frameHeader zfh;
-
- /* Extract Frame Header */
- { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
- if (ZSTD_isError(ret))
- return ZSTD_errorFrameSizeInfo(ret);
- if (ret > 0)
- return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
- }
-
- ip += zfh.headerSize;
- remainingSize -= zfh.headerSize;
-
- /* Iterate over each block */
- while (1) {
- blockProperties_t blockProperties;
- size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
- if (ZSTD_isError(cBlockSize))
- return ZSTD_errorFrameSizeInfo(cBlockSize);
-
- if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
- return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
-
- ip += ZSTD_blockHeaderSize + cBlockSize;
- remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
- nbBlocks++;
-
- if (blockProperties.lastBlock) break;
- }
-
- /* Final frame content checksum */
- if (zfh.checksumFlag) {
- if (remainingSize < 4)
- return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
- ip += 4;
- }
-
- frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
- frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
- ? zfh.frameContentSize
- : nbBlocks * zfh.blockSizeMax;
- return frameSizeInfo;
- }
-}
-
-/** ZSTD_findFrameCompressedSize() :
- * compatible with legacy mode
- * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
- * `srcSize` must be at least as large as the frame contained
- * @return : the compressed size of the frame starting at `src` */
-size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
-{
- ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
- return frameSizeInfo.compressedSize;
-}
-
-/** ZSTD_decompressBound() :
- * compatible with legacy mode
- * `src` must point to the start of a ZSTD frame or a skippeable frame
- * `srcSize` must be at least as large as the frame contained
- * @return : the maximum decompressed size of the compressed source
- */
-unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
-{
- unsigned long long bound = 0;
- /* Iterate over each frame */
- while (srcSize > 0) {
- ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
- size_t const compressedSize = frameSizeInfo.compressedSize;
- unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
- if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
- return ZSTD_CONTENTSIZE_ERROR;
- assert(srcSize >= compressedSize);
- src = (const BYTE*)src + compressedSize;
- srcSize -= compressedSize;
- bound += decompressedBound;
- }
- return bound;
-}
-
-
-/*-*************************************************************
- * Frame decoding
- ***************************************************************/
-
-/** ZSTD_insertBlock() :
- * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
-size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
-{
- DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize);
- ZSTD_checkContinuity(dctx, blockStart, blockSize);
- dctx->previousDstEnd = (const char*)blockStart + blockSize;
- return blockSize;
-}
-
-
-static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
- const void* src, size_t srcSize)
-{
- DEBUGLOG(5, "ZSTD_copyRawBlock");
- RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
- if (dst == NULL) {
- if (srcSize == 0) return 0;
- RETURN_ERROR(dstBuffer_null, "");
- }
- ZSTD_memcpy(dst, src, srcSize);
- return srcSize;
-}
-
-static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
- BYTE b,
- size_t regenSize)
-{
- RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
- if (dst == NULL) {
- if (regenSize == 0) return 0;
- RETURN_ERROR(dstBuffer_null, "");
- }
- ZSTD_memset(dst, b, regenSize);
- return regenSize;
-}
-
-static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
-{
-#if ZSTD_TRACE
- if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) {
- ZSTD_Trace trace;
- ZSTD_memset(&trace, 0, sizeof(trace));
- trace.version = ZSTD_VERSION_NUMBER;
- trace.streaming = streaming;
- if (dctx->ddict) {
- trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict);
- trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict);
- trace.dictionaryIsCold = dctx->ddictIsCold;
- }
- trace.uncompressedSize = (size_t)uncompressedSize;
- trace.compressedSize = (size_t)compressedSize;
- trace.dctx = dctx;
- ZSTD_trace_decompress_end(dctx->traceCtx, &trace);
- }
-#else
- (void)dctx;
- (void)uncompressedSize;
- (void)compressedSize;
- (void)streaming;
-#endif
-}
-
-
-/*! ZSTD_decompressFrame() :
- * @dctx must be properly initialized
- * will update *srcPtr and *srcSizePtr,
- * to make *srcPtr progress by one frame. */
-static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void** srcPtr, size_t *srcSizePtr)
-{
- const BYTE* const istart = (const BYTE*)(*srcPtr);
- const BYTE* ip = istart;
- BYTE* const ostart = (BYTE*)dst;
- BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
- BYTE* op = ostart;
- size_t remainingSrcSize = *srcSizePtr;
-
- DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr);
-
- /* check */
- RETURN_ERROR_IF(
- remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize,
- srcSize_wrong, "");
-
- /* Frame Header */
- { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal(
- ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format);
- if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
- RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
- srcSize_wrong, "");
- FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , "");
- ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
- }
-
- /* Loop on each block */
- while (1) {
- size_t decodedSize;
- blockProperties_t blockProperties;
- size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties);
- if (ZSTD_isError(cBlockSize)) return cBlockSize;
-
- ip += ZSTD_blockHeaderSize;
- remainingSrcSize -= ZSTD_blockHeaderSize;
- RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, "");
-
- switch(blockProperties.blockType)
- {
- case bt_compressed:
- decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1, not_streaming);
- break;
- case bt_raw :
- decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize);
- break;
- case bt_rle :
- decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize);
- break;
- case bt_reserved :
- default:
- RETURN_ERROR(corruption_detected, "invalid block type");
- }
-
- if (ZSTD_isError(decodedSize)) return decodedSize;
- if (dctx->validateChecksum)
- XXH64_update(&dctx->xxhState, op, decodedSize);
- if (decodedSize != 0)
- op += decodedSize;
- assert(ip != NULL);
- ip += cBlockSize;
- remainingSrcSize -= cBlockSize;
- if (blockProperties.lastBlock) break;
- }
-
- if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) {
- RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize,
- corruption_detected, "");
- }
- if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
- RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, "");
- if (!dctx->forceIgnoreChecksum) {
- U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
- U32 checkRead;
- checkRead = MEM_readLE32(ip);
- RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, "");
- }
- ip += 4;
- remainingSrcSize -= 4;
- }
- ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
- /* Allow caller to get size read */
- *srcPtr = ip;
- *srcSizePtr = remainingSrcSize;
- return (size_t)(op-ostart);
-}
-
-static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict, size_t dictSize,
- const ZSTD_DDict* ddict)
-{
- void* const dststart = dst;
- int moreThan1Frame = 0;
-
- DEBUGLOG(5, "ZSTD_decompressMultiFrame");
- assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */
-
- if (ddict) {
- dict = ZSTD_DDict_dictContent(ddict);
- dictSize = ZSTD_DDict_dictSize(ddict);
- }
-
- while (srcSize >= ZSTD_startingInputLength(dctx->format)) {
-
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
- if (ZSTD_isLegacy(src, srcSize)) {
- size_t decodedSize;
- size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
- if (ZSTD_isError(frameSize)) return frameSize;
- RETURN_ERROR_IF(dctx->staticSize, memory_allocation,
- "legacy support is not compatible with static dctx");
-
- decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
- if (ZSTD_isError(decodedSize)) return decodedSize;
-
- assert(decodedSize <= dstCapacity);
- dst = (BYTE*)dst + decodedSize;
- dstCapacity -= decodedSize;
-
- src = (const BYTE*)src + frameSize;
- srcSize -= frameSize;
-
- continue;
- }
-#endif
-
- { U32 const magicNumber = MEM_readLE32(src);
- DEBUGLOG(4, "reading magic number %08X (expecting %08X)",
- (unsigned)magicNumber, ZSTD_MAGICNUMBER);
- if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
- size_t const skippableSize = readSkippableFrameSize(src, srcSize);
- FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed");
- assert(skippableSize <= srcSize);
-
- src = (const BYTE *)src + skippableSize;
- srcSize -= skippableSize;
- continue;
- } }
-
- if (ddict) {
- /* we were called from ZSTD_decompress_usingDDict */
- FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), "");
- } else {
- /* this will initialize correctly with no dict if dict == NULL, so
- * use this in all cases but ddict */
- FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), "");
- }
- ZSTD_checkContinuity(dctx, dst, dstCapacity);
-
- { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
- &src, &srcSize);
- RETURN_ERROR_IF(
- (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown)
- && (moreThan1Frame==1),
- srcSize_wrong,
- "At least one frame successfully completed, "
- "but following bytes are garbage: "
- "it's more likely to be a srcSize error, "
- "specifying more input bytes than size of frame(s). "
- "Note: one could be unlucky, it might be a corruption error instead, "
- "happening right at the place where we expect zstd magic bytes. "
- "But this is _much_ less likely than a srcSize field error.");
- if (ZSTD_isError(res)) return res;
- assert(res <= dstCapacity);
- if (res != 0)
- dst = (BYTE*)dst + res;
- dstCapacity -= res;
- }
- moreThan1Frame = 1;
- } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
-
- RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed");
-
- return (size_t)((BYTE*)dst - (BYTE*)dststart);
-}
-
-size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict, size_t dictSize)
-{
- return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
-}
-
-
-static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx)
-{
- switch (dctx->dictUses) {
- default:
- assert(0 /* Impossible */);
- ZSTD_FALLTHROUGH;
- case ZSTD_dont_use:
- ZSTD_clearDict(dctx);
- return NULL;
- case ZSTD_use_indefinitely:
- return dctx->ddict;
- case ZSTD_use_once:
- dctx->dictUses = ZSTD_dont_use;
- return dctx->ddict;
- }
-}
-
-size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
-{
- return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx));
-}
-
-
-size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
-{
-#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
- size_t regenSize;
- ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem);
- RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!");
- regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
- ZSTD_freeDCtx(dctx);
- return regenSize;
-#else /* stack mode */
- ZSTD_DCtx dctx;
- ZSTD_initDCtx_internal(&dctx);
- return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
-#endif
-}
-
-
-/*-**************************************
-* Advanced Streaming Decompression API
-* Bufferless and synchronous
-****************************************/
-size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; }
-
-/**
- * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed,
- * we allow taking a partial block as the input. Currently only raw uncompressed blocks can
- * be streamed.
- *
- * For blocks that can be streamed, this allows us to reduce the latency until we produce
- * output, and avoid copying the input.
- *
- * @param inputSize - The total amount of input that the caller currently has.
- */
-static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) {
- if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock))
- return dctx->expected;
- if (dctx->bType != bt_raw)
- return dctx->expected;
- return BOUNDED(1, inputSize, dctx->expected);
-}
-
-ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
- switch(dctx->stage)
- {
- default: /* should not happen */
- assert(0);
- ZSTD_FALLTHROUGH;
- case ZSTDds_getFrameHeaderSize:
- ZSTD_FALLTHROUGH;
- case ZSTDds_decodeFrameHeader:
- return ZSTDnit_frameHeader;
- case ZSTDds_decodeBlockHeader:
- return ZSTDnit_blockHeader;
- case ZSTDds_decompressBlock:
- return ZSTDnit_block;
- case ZSTDds_decompressLastBlock:
- return ZSTDnit_lastBlock;
- case ZSTDds_checkChecksum:
- return ZSTDnit_checksum;
- case ZSTDds_decodeSkippableHeader:
- ZSTD_FALLTHROUGH;
- case ZSTDds_skipFrame:
- return ZSTDnit_skippableFrame;
- }
-}
-
-static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; }
-
-/** ZSTD_decompressContinue() :
- * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress())
- * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
- * or an error code, which can be tested using ZSTD_isError() */
-size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
-{
- DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);
- /* Sanity check */
- RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");
- ZSTD_checkContinuity(dctx, dst, dstCapacity);
-
- dctx->processedCSize += srcSize;
-
- switch (dctx->stage)
- {
- case ZSTDds_getFrameHeaderSize :
- assert(src != NULL);
- if (dctx->format == ZSTD_f_zstd1) { /* allows header */
- assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */
- if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
- ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
- dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */
- dctx->stage = ZSTDds_decodeSkippableHeader;
- return 0;
- } }
- dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format);
- if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
- ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
- dctx->expected = dctx->headerSize - srcSize;
- dctx->stage = ZSTDds_decodeFrameHeader;
- return 0;
-
- case ZSTDds_decodeFrameHeader:
- assert(src != NULL);
- ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
- FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), "");
- dctx->expected = ZSTD_blockHeaderSize;
- dctx->stage = ZSTDds_decodeBlockHeader;
- return 0;
-
- case ZSTDds_decodeBlockHeader:
- { blockProperties_t bp;
- size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
- if (ZSTD_isError(cBlockSize)) return cBlockSize;
- RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum");
- dctx->expected = cBlockSize;
- dctx->bType = bp.blockType;
- dctx->rleSize = bp.origSize;
- if (cBlockSize) {
- dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock;
- return 0;
- }
- /* empty block */
- if (bp.lastBlock) {
- if (dctx->fParams.checksumFlag) {
- dctx->expected = 4;
- dctx->stage = ZSTDds_checkChecksum;
- } else {
- dctx->expected = 0; /* end of frame */
- dctx->stage = ZSTDds_getFrameHeaderSize;
- }
- } else {
- dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */
- dctx->stage = ZSTDds_decodeBlockHeader;
- }
- return 0;
- }
-
- case ZSTDds_decompressLastBlock:
- case ZSTDds_decompressBlock:
- DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock");
- { size_t rSize;
- switch(dctx->bType)
- {
- case bt_compressed:
- DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed");
- rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming);
- dctx->expected = 0; /* Streaming not supported */
- break;
- case bt_raw :
- assert(srcSize <= dctx->expected);
- rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
- FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed");
- assert(rSize == srcSize);
- dctx->expected -= rSize;
- break;
- case bt_rle :
- rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize);
- dctx->expected = 0; /* Streaming not supported */
- break;
- case bt_reserved : /* should never happen */
- default:
- RETURN_ERROR(corruption_detected, "invalid block type");
- }
- FORWARD_IF_ERROR(rSize, "");
- RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum");
- DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
- dctx->decodedSize += rSize;
- if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize);
- dctx->previousDstEnd = (char*)dst + rSize;
-
- /* Stay on the same stage until we are finished streaming the block. */
- if (dctx->expected > 0) {
- return rSize;
- }
-
- if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
- DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize);
- RETURN_ERROR_IF(
- dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
- && dctx->decodedSize != dctx->fParams.frameContentSize,
- corruption_detected, "");
- if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
- dctx->expected = 4;
- dctx->stage = ZSTDds_checkChecksum;
- } else {
- ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
- dctx->expected = 0; /* ends here */
- dctx->stage = ZSTDds_getFrameHeaderSize;
- }
- } else {
- dctx->stage = ZSTDds_decodeBlockHeader;
- dctx->expected = ZSTD_blockHeaderSize;
- }
- return rSize;
- }
-
- case ZSTDds_checkChecksum:
- assert(srcSize == 4); /* guaranteed by dctx->expected */
- {
- if (dctx->validateChecksum) {
- U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
- U32 const check32 = MEM_readLE32(src);
- DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
- RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
- }
- ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
- dctx->expected = 0;
- dctx->stage = ZSTDds_getFrameHeaderSize;
- return 0;
- }
-
- case ZSTDds_decodeSkippableHeader:
- assert(src != NULL);
- assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE);
- ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */
- dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */
- dctx->stage = ZSTDds_skipFrame;
- return 0;
-
- case ZSTDds_skipFrame:
- dctx->expected = 0;
- dctx->stage = ZSTDds_getFrameHeaderSize;
- return 0;
-
- default:
- assert(0); /* impossible */
- RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */
- }
-}
-
-
-static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
-{
- dctx->dictEnd = dctx->previousDstEnd;
- dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
- dctx->prefixStart = dict;
- dctx->previousDstEnd = (const char*)dict + dictSize;
-#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
- dctx->dictContentBeginForFuzzing = dctx->prefixStart;
- dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
-#endif
- return 0;
-}
-
-/*! ZSTD_loadDEntropy() :
- * dict : must point at beginning of a valid zstd dictionary.
- * @return : size of entropy tables read */
-size_t
-ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
- const void* const dict, size_t const dictSize)
-{
- const BYTE* dictPtr = (const BYTE*)dict;
- const BYTE* const dictEnd = dictPtr + dictSize;
-
- RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small");
- assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */
- dictPtr += 8; /* skip header = magic + dictID */
-
- ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable));
- ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable));
- ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE);
- { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */
- size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable);
-#ifdef HUF_FORCE_DECOMPRESS_X1
- /* in minimal huffman, we always use X1 variants */
- size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable,
- dictPtr, dictEnd - dictPtr,
- workspace, workspaceSize);
-#else
- size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
- dictPtr, (size_t)(dictEnd - dictPtr),
- workspace, workspaceSize);
-#endif
- RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
- dictPtr += hSize;
- }
-
- { short offcodeNCount[MaxOff+1];
- unsigned offcodeMaxValue = MaxOff, offcodeLog;
- size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr));
- RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
- RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, "");
- RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
- ZSTD_buildFSETable( entropy->OFTable,
- offcodeNCount, offcodeMaxValue,
- OF_base, OF_bits,
- offcodeLog,
- entropy->workspace, sizeof(entropy->workspace),
- /* bmi2 */0);
- dictPtr += offcodeHeaderSize;
- }
-
- { short matchlengthNCount[MaxML+1];
- unsigned matchlengthMaxValue = MaxML, matchlengthLog;
- size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
- RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
- RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, "");
- RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
- ZSTD_buildFSETable( entropy->MLTable,
- matchlengthNCount, matchlengthMaxValue,
- ML_base, ML_bits,
- matchlengthLog,
- entropy->workspace, sizeof(entropy->workspace),
- /* bmi2 */ 0);
- dictPtr += matchlengthHeaderSize;
- }
-
- { short litlengthNCount[MaxLL+1];
- unsigned litlengthMaxValue = MaxLL, litlengthLog;
- size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
- RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
- RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, "");
- RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
- ZSTD_buildFSETable( entropy->LLTable,
- litlengthNCount, litlengthMaxValue,
- LL_base, LL_bits,
- litlengthLog,
- entropy->workspace, sizeof(entropy->workspace),
- /* bmi2 */ 0);
- dictPtr += litlengthHeaderSize;
- }
-
- RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, "");
- { int i;
- size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
- for (i=0; i<3; i++) {
- U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
- RETURN_ERROR_IF(rep==0 || rep > dictContentSize,
- dictionary_corrupted, "");
- entropy->rep[i] = rep;
- } }
-
- return (size_t)(dictPtr - (const BYTE*)dict);
-}
-
-static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
-{
- if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
- { U32 const magic = MEM_readLE32(dict);
- if (magic != ZSTD_MAGIC_DICTIONARY) {
- return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
- } }
- dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
-
- /* load entropy tables */
- { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize);
- RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, "");
- dict = (const char*)dict + eSize;
- dictSize -= eSize;
- }
- dctx->litEntropy = dctx->fseEntropy = 1;
-
- /* reference dictionary content */
- return ZSTD_refDictContent(dctx, dict, dictSize);
-}
-
-size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
-{
- assert(dctx != NULL);
-#if ZSTD_TRACE
- dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0;
-#endif
- dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */
- dctx->stage = ZSTDds_getFrameHeaderSize;
- dctx->processedCSize = 0;
- dctx->decodedSize = 0;
- dctx->previousDstEnd = NULL;
- dctx->prefixStart = NULL;
- dctx->virtualStart = NULL;
- dctx->dictEnd = NULL;
- dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
- dctx->litEntropy = dctx->fseEntropy = 0;
- dctx->dictID = 0;
- dctx->bType = bt_reserved;
- ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
- ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
- dctx->LLTptr = dctx->entropy.LLTable;
- dctx->MLTptr = dctx->entropy.MLTable;
- dctx->OFTptr = dctx->entropy.OFTable;
- dctx->HUFptr = dctx->entropy.hufTable;
- return 0;
-}
-
-size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
-{
- FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
- if (dict && dictSize)
- RETURN_ERROR_IF(
- ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)),
- dictionary_corrupted, "");
- return 0;
-}
-
-
-/* ====== ZSTD_DDict ====== */
-
-size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
-{
- DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict");
- assert(dctx != NULL);
- if (ddict) {
- const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict);
- size_t const dictSize = ZSTD_DDict_dictSize(ddict);
- const void* const dictEnd = dictStart + dictSize;
- dctx->ddictIsCold = (dctx->dictEnd != dictEnd);
- DEBUGLOG(4, "DDict is %s",
- dctx->ddictIsCold ? "~cold~" : "hot!");
- }
- FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
- if (ddict) { /* NULL ddict is equivalent to no dictionary */
- ZSTD_copyDDictParameters(dctx, ddict);
- }
- return 0;
-}
-
-/*! ZSTD_getDictID_fromDict() :
- * Provides the dictID stored within dictionary.
- * if @return == 0, the dictionary is not conformant with Zstandard specification.
- * It can still be loaded, but as a content-only dictionary. */
-unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
-{
- if (dictSize < 8) return 0;
- if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
- return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
-}
-
-/*! ZSTD_getDictID_fromFrame() :
- * Provides the dictID required to decompress frame stored within `src`.
- * If @return == 0, the dictID could not be decoded.
- * This could for one of the following reasons :
- * - The frame does not require a dictionary (most common case).
- * - The frame was built with dictID intentionally removed.
- * Needed dictionary is a hidden information.
- * Note : this use case also happens when using a non-conformant dictionary.
- * - `srcSize` is too small, and as a result, frame header could not be decoded.
- * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`.
- * - This is not a Zstandard frame.
- * When identifying the exact failure cause, it's possible to use
- * ZSTD_getFrameHeader(), which will provide a more precise error code. */
-unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
-{
- ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 };
- size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
- if (ZSTD_isError(hError)) return 0;
- return zfp.dictID;
-}
-
-
-/*! ZSTD_decompress_usingDDict() :
-* Decompression using a pre-digested Dictionary
-* Use dictionary without significant overhead. */
-size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTD_DDict* ddict)
-{
- /* pass content and size in case legacy frames are encountered */
- return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
- NULL, 0,
- ddict);
-}
-
-
-/*=====================================
-* Streaming decompression
-*====================================*/
-
-ZSTD_DStream* ZSTD_createDStream(void)
-{
- DEBUGLOG(3, "ZSTD_createDStream");
- return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
-}
-
-ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
-{
- return ZSTD_initStaticDCtx(workspace, workspaceSize);
-}
-
-ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
-{
- return ZSTD_createDCtx_internal(customMem);
-}
-
-size_t ZSTD_freeDStream(ZSTD_DStream* zds)
-{
- return ZSTD_freeDCtx(zds);
-}
-
-
-/* *** Initialization *** */
-
-size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
-size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
-
-size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
- const void* dict, size_t dictSize,
- ZSTD_dictLoadMethod_e dictLoadMethod,
- ZSTD_dictContentType_e dictContentType)
-{
- RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
- ZSTD_clearDict(dctx);
- if (dict && dictSize != 0) {
- dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
- RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!");
- dctx->ddict = dctx->ddictLocal;
- dctx->dictUses = ZSTD_use_indefinitely;
- }
- return 0;
-}
-
-size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
-{
- return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
-}
-
-size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
-{
- return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
-}
-
-size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
-{
- FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), "");
- dctx->dictUses = ZSTD_use_once;
- return 0;
-}
-
-size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize)
-{
- return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent);
-}
-
-
-/* ZSTD_initDStream_usingDict() :
- * return : expected size, aka ZSTD_startingInputLength().
- * this function cannot fail */
-size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
-{
- DEBUGLOG(4, "ZSTD_initDStream_usingDict");
- FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , "");
- FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , "");
- return ZSTD_startingInputLength(zds->format);
-}
-
-/* note : this variant can't fail */
-size_t ZSTD_initDStream(ZSTD_DStream* zds)
-{
- DEBUGLOG(4, "ZSTD_initDStream");
- return ZSTD_initDStream_usingDDict(zds, NULL);
-}
-
-/* ZSTD_initDStream_usingDDict() :
- * ddict will just be referenced, and must outlive decompression session
- * this function cannot fail */
-size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
-{
- FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");
- FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");
- return ZSTD_startingInputLength(dctx->format);
-}
-
-/* ZSTD_resetDStream() :
- * return : expected size, aka ZSTD_startingInputLength().
- * this function cannot fail */
-size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
-{
- FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");
- return ZSTD_startingInputLength(dctx->format);
-}
-
-
-size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
-{
- RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
- ZSTD_clearDict(dctx);
- if (ddict) {
- dctx->ddict = ddict;
- dctx->dictUses = ZSTD_use_indefinitely;
- if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) {
- if (dctx->ddictSet == NULL) {
- dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
- if (!dctx->ddictSet) {
- RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!");
- }
- }
- assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */
- FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), "");
- }
- }
- return 0;
-}
-
-/* ZSTD_DCtx_setMaxWindowSize() :
- * note : no direct equivalence in ZSTD_DCtx_setParameter,
- * since this version sets windowSize, and the other sets windowLog */
-size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
-{
- ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
- size_t const min = (size_t)1 << bounds.lowerBound;
- size_t const max = (size_t)1 << bounds.upperBound;
- RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
- RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, "");
- RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, "");
- dctx->maxWindowSize = maxWindowSize;
- return 0;
-}
-
-size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
-{
- return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format);
-}
-
-ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
-{
- ZSTD_bounds bounds = { 0, 0, 0 };
- switch(dParam) {
- case ZSTD_d_windowLogMax:
- bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN;
- bounds.upperBound = ZSTD_WINDOWLOG_MAX;
- return bounds;
- case ZSTD_d_format:
- bounds.lowerBound = (int)ZSTD_f_zstd1;
- bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
- ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
- return bounds;
- case ZSTD_d_stableOutBuffer:
- bounds.lowerBound = (int)ZSTD_bm_buffered;
- bounds.upperBound = (int)ZSTD_bm_stable;
- return bounds;
- case ZSTD_d_forceIgnoreChecksum:
- bounds.lowerBound = (int)ZSTD_d_validateChecksum;
- bounds.upperBound = (int)ZSTD_d_ignoreChecksum;
- return bounds;
- case ZSTD_d_refMultipleDDicts:
- bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
- bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
- return bounds;
- default:;
- }
- bounds.error = ERROR(parameter_unsupported);
- return bounds;
-}
-
-/* ZSTD_dParam_withinBounds:
- * @return 1 if value is within dParam bounds,
- * 0 otherwise */
-static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
-{
- ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam);
- if (ZSTD_isError(bounds.error)) return 0;
- if (value < bounds.lowerBound) return 0;
- if (value > bounds.upperBound) return 0;
- return 1;
-}
-
-#define CHECK_DBOUNDS(p,v) { \
- RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \
-}
-
-size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value)
-{
- switch (param) {
- case ZSTD_d_windowLogMax:
- *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize);
- return 0;
- case ZSTD_d_format:
- *value = (int)dctx->format;
- return 0;
- case ZSTD_d_stableOutBuffer:
- *value = (int)dctx->outBufferMode;
- return 0;
- case ZSTD_d_forceIgnoreChecksum:
- *value = (int)dctx->forceIgnoreChecksum;
- return 0;
- case ZSTD_d_refMultipleDDicts:
- *value = (int)dctx->refMultipleDDicts;
- return 0;
- default:;
- }
- RETURN_ERROR(parameter_unsupported, "");
-}
-
-size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
-{
- RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
- switch(dParam) {
- case ZSTD_d_windowLogMax:
- if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
- CHECK_DBOUNDS(ZSTD_d_windowLogMax, value);
- dctx->maxWindowSize = ((size_t)1) << value;
- return 0;
- case ZSTD_d_format:
- CHECK_DBOUNDS(ZSTD_d_format, value);
- dctx->format = (ZSTD_format_e)value;
- return 0;
- case ZSTD_d_stableOutBuffer:
- CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value);
- dctx->outBufferMode = (ZSTD_bufferMode_e)value;
- return 0;
- case ZSTD_d_forceIgnoreChecksum:
- CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value);
- dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value;
- return 0;
- case ZSTD_d_refMultipleDDicts:
- CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
- if (dctx->staticSize != 0) {
- RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!");
- }
- dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
- return 0;
- default:;
- }
- RETURN_ERROR(parameter_unsupported, "");
-}
-
-size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
-{
- if ( (reset == ZSTD_reset_session_only)
- || (reset == ZSTD_reset_session_and_parameters) ) {
- dctx->streamStage = zdss_init;
- dctx->noForwardProgress = 0;
- }
- if ( (reset == ZSTD_reset_parameters)
- || (reset == ZSTD_reset_session_and_parameters) ) {
- RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
- ZSTD_clearDict(dctx);
- ZSTD_DCtx_resetParameters(dctx);
- }
- return 0;
-}
-
-
-size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx)
-{
- return ZSTD_sizeof_DCtx(dctx);
-}
-
-size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize)
-{
- size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
- /* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/
- unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2);
- unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
- size_t const minRBSize = (size_t) neededSize;
- RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
- frameParameter_windowTooLarge, "");
- return minRBSize;
-}
-
-size_t ZSTD_estimateDStreamSize(size_t windowSize)
-{
- size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
- size_t const inBuffSize = blockSize; /* no block can be larger */
- size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN);
- return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
-}
-
-size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
-{
- U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */
- ZSTD_frameHeader zfh;
- size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
- if (ZSTD_isError(err)) return err;
- RETURN_ERROR_IF(err>0, srcSize_wrong, "");
- RETURN_ERROR_IF(zfh.windowSize > windowSizeMax,
- frameParameter_windowTooLarge, "");
- return ZSTD_estimateDStreamSize((size_t)zfh.windowSize);
-}
-
-
-/* ***** Decompression ***** */
-
-static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
-{
- return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR;
-}
-
-static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
-{
- if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize))
- zds->oversizedDuration++;
- else
- zds->oversizedDuration = 0;
-}
-
-static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds)
-{
- return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION;
-}
-
-/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */
-static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output)
-{
- ZSTD_outBuffer const expect = zds->expectedOutBuffer;
- /* No requirement when ZSTD_obm_stable is not enabled. */
- if (zds->outBufferMode != ZSTD_bm_stable)
- return 0;
- /* Any buffer is allowed in zdss_init, this must be the same for every other call until
- * the context is reset.
- */
- if (zds->streamStage == zdss_init)
- return 0;
- /* The buffer must match our expectation exactly. */
- if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size)
- return 0;
- RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!");
-}
-
-/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream()
- * and updates the stage and the output buffer state. This call is extracted so it can be
- * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode.
- * NOTE: You must break after calling this function since the streamStage is modified.
- */
-static size_t ZSTD_decompressContinueStream(
- ZSTD_DStream* zds, char** op, char* oend,
- void const* src, size_t srcSize) {
- int const isSkipFrame = ZSTD_isSkipFrame(zds);
- if (zds->outBufferMode == ZSTD_bm_buffered) {
- size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart;
- size_t const decodedSize = ZSTD_decompressContinue(zds,
- zds->outBuff + zds->outStart, dstSize, src, srcSize);
- FORWARD_IF_ERROR(decodedSize, "");
- if (!decodedSize && !isSkipFrame) {
- zds->streamStage = zdss_read;
- } else {
- zds->outEnd = zds->outStart + decodedSize;
- zds->streamStage = zdss_flush;
- }
- } else {
- /* Write directly into the output buffer */
- size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op);
- size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize);
- FORWARD_IF_ERROR(decodedSize, "");
- *op += decodedSize;
- /* Flushing is not needed. */
- zds->streamStage = zdss_read;
- assert(*op <= oend);
- assert(zds->outBufferMode == ZSTD_bm_stable);
- }
- return 0;
-}
-
-size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
-{
- const char* const src = (const char*)input->src;
- const char* const istart = input->pos != 0 ? src + input->pos : src;
- const char* const iend = input->size != 0 ? src + input->size : src;
- const char* ip = istart;
- char* const dst = (char*)output->dst;
- char* const ostart = output->pos != 0 ? dst + output->pos : dst;
- char* const oend = output->size != 0 ? dst + output->size : dst;
- char* op = ostart;
- U32 someMoreWork = 1;
-
- DEBUGLOG(5, "ZSTD_decompressStream");
- RETURN_ERROR_IF(
- input->pos > input->size,
- srcSize_wrong,
- "forbidden. in: pos: %u vs size: %u",
- (U32)input->pos, (U32)input->size);
- RETURN_ERROR_IF(
- output->pos > output->size,
- dstSize_tooSmall,
- "forbidden. out: pos: %u vs size: %u",
- (U32)output->pos, (U32)output->size);
- DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
- FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), "");
-
- while (someMoreWork) {
- switch(zds->streamStage)
- {
- case zdss_init :
- DEBUGLOG(5, "stage zdss_init => transparent reset ");
- zds->streamStage = zdss_loadHeader;
- zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
- zds->legacyVersion = 0;
-#endif
- zds->hostageByte = 0;
- zds->expectedOutBuffer = *output;
- ZSTD_FALLTHROUGH;
-
- case zdss_loadHeader :
- DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip));
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
- if (zds->legacyVersion) {
- RETURN_ERROR_IF(zds->staticSize, memory_allocation,
- "legacy support is incompatible with static dctx");
- { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
- if (hint==0) zds->streamStage = zdss_init;
- return hint;
- } }
-#endif
- { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
- if (zds->refMultipleDDicts && zds->ddictSet) {
- ZSTD_DCtx_selectFrameDDict(zds);
- }
- DEBUGLOG(5, "header size : %u", (U32)hSize);
- if (ZSTD_isError(hSize)) {
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
- U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
- if (legacyVersion) {
- ZSTD_DDict const* const ddict = ZSTD_getDDict(zds);
- const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL;
- size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0;
- DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
- RETURN_ERROR_IF(zds->staticSize, memory_allocation,
- "legacy support is incompatible with static dctx");
- FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext,
- zds->previousLegacyVersion, legacyVersion,
- dict, dictSize), "");
- zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
- { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
- if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */
- return hint;
- } }
-#endif
- return hSize; /* error */
- }
- if (hSize != 0) { /* need more input */
- size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
- size_t const remainingInput = (size_t)(iend-ip);
- assert(iend >= ip);
- if (toLoad > remainingInput) { /* not enough input to load full header */
- if (remainingInput > 0) {
- ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
- zds->lhSize += remainingInput;
- }
- input->pos = input->size;
- return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
- }
- assert(ip != NULL);
- ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
- break;
- } }
-
- /* check for single-pass mode opportunity */
- if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
- && zds->fParams.frameType != ZSTD_skippableFrame
- && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
- size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart));
- if (cSize <= (size_t)(iend-istart)) {
- /* shortcut : using single-pass mode */
- size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds));
- if (ZSTD_isError(decompressedSize)) return decompressedSize;
- DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
- ip = istart + cSize;
- op += decompressedSize;
- zds->expected = 0;
- zds->streamStage = zdss_init;
- someMoreWork = 0;
- break;
- } }
-
- /* Check output buffer is large enough for ZSTD_odm_stable. */
- if (zds->outBufferMode == ZSTD_bm_stable
- && zds->fParams.frameType != ZSTD_skippableFrame
- && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
- && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) {
- RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small");
- }
-
- /* Consume header (see ZSTDds_decodeFrameHeader) */
- DEBUGLOG(4, "Consume header");
- FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), "");
-
- if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
- zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
- zds->stage = ZSTDds_skipFrame;
- } else {
- FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), "");
- zds->expected = ZSTD_blockHeaderSize;
- zds->stage = ZSTDds_decodeBlockHeader;
- }
-
- /* control buffer memory usage */
- DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)",
- (U32)(zds->fParams.windowSize >>10),
- (U32)(zds->maxWindowSize >> 10) );
- zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
- RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize,
- frameParameter_windowTooLarge, "");
-
- /* Adapt buffer sizes to frame header instructions */
- { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
- size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered
- ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize)
- : 0;
-
- ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize);
-
- { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize);
- int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds);
-
- if (tooSmall || tooLarge) {
- size_t const bufferSize = neededInBuffSize + neededOutBuffSize;
- DEBUGLOG(4, "inBuff : from %u to %u",
- (U32)zds->inBuffSize, (U32)neededInBuffSize);
- DEBUGLOG(4, "outBuff : from %u to %u",
- (U32)zds->outBuffSize, (U32)neededOutBuffSize);
- if (zds->staticSize) { /* static DCtx */
- DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
- assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */
- RETURN_ERROR_IF(
- bufferSize > zds->staticSize - sizeof(ZSTD_DCtx),
- memory_allocation, "");
- } else {
- ZSTD_customFree(zds->inBuff, zds->customMem);
- zds->inBuffSize = 0;
- zds->outBuffSize = 0;
- zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem);
- RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, "");
- }
- zds->inBuffSize = neededInBuffSize;
- zds->outBuff = zds->inBuff + zds->inBuffSize;
- zds->outBuffSize = neededOutBuffSize;
- } } }
- zds->streamStage = zdss_read;
- ZSTD_FALLTHROUGH;
-
- case zdss_read:
- DEBUGLOG(5, "stage zdss_read");
- { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip));
- DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
- if (neededInSize==0) { /* end of frame */
- zds->streamStage = zdss_init;
- someMoreWork = 0;
- break;
- }
- if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
- FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), "");
- ip += neededInSize;
- /* Function modifies the stage so we must break */
- break;
- } }
- if (ip==iend) { someMoreWork = 0; break; } /* no more input */
- zds->streamStage = zdss_load;
- ZSTD_FALLTHROUGH;
-
- case zdss_load:
- { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
- size_t const toLoad = neededInSize - zds->inPos;
- int const isSkipFrame = ZSTD_isSkipFrame(zds);
- size_t loadedSize;
- /* At this point we shouldn't be decompressing a block that we can stream. */
- assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip));
- if (isSkipFrame) {
- loadedSize = MIN(toLoad, (size_t)(iend-ip));
- } else {
- RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos,
- corruption_detected,
- "should never happen");
- loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip));
- }
- ip += loadedSize;
- zds->inPos += loadedSize;
- if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
-
- /* decode loaded input */
- zds->inPos = 0; /* input is consumed */
- FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), "");
- /* Function modifies the stage so we must break */
- break;
- }
- case zdss_flush:
- { size_t const toFlushSize = zds->outEnd - zds->outStart;
- size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
- op += flushedSize;
- zds->outStart += flushedSize;
- if (flushedSize == toFlushSize) { /* flush completed */
- zds->streamStage = zdss_read;
- if ( (zds->outBuffSize < zds->fParams.frameContentSize)
- && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
- DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)",
- (int)(zds->outBuffSize - zds->outStart),
- (U32)zds->fParams.blockSizeMax);
- zds->outStart = zds->outEnd = 0;
- }
- break;
- } }
- /* cannot complete flush */
- someMoreWork = 0;
- break;
-
- default:
- assert(0); /* impossible */
- RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */
- } }
-
- /* result */
- input->pos = (size_t)(ip - (const char*)(input->src));
- output->pos = (size_t)(op - (char*)(output->dst));
-
- /* Update the expected output buffer for ZSTD_obm_stable. */
- zds->expectedOutBuffer = *output;
-
- if ((ip==istart) && (op==ostart)) { /* no forward progress */
- zds->noForwardProgress ++;
- if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
- RETURN_ERROR_IF(op==oend, dstSize_tooSmall, "");
- RETURN_ERROR_IF(ip==iend, srcSize_wrong, "");
- assert(0);
- }
- } else {
- zds->noForwardProgress = 0;
- }
- { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds);
- if (!nextSrcSizeHint) { /* frame fully decoded */
- if (zds->outEnd == zds->outStart) { /* output fully flushed */
- if (zds->hostageByte) {
- if (input->pos >= input->size) {
- /* can't release hostage (not present) */
- zds->streamStage = zdss_read;
- return 1;
- }
- input->pos++; /* release hostage */
- } /* zds->hostageByte */
- return 0;
- } /* zds->outEnd == zds->outStart */
- if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
- input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
- zds->hostageByte=1;
- }
- return 1;
- } /* nextSrcSizeHint==0 */
- nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */
- assert(zds->inPos <= nextSrcSizeHint);
- nextSrcSizeHint -= zds->inPos; /* part already loaded*/
- return nextSrcSizeHint;
- }
-}
-
-size_t ZSTD_decompressStream_simpleArgs (
- ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity, size_t* dstPos,
- const void* src, size_t srcSize, size_t* srcPos)
-{
- ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
- ZSTD_inBuffer input = { src, srcSize, *srcPos };
- /* ZSTD_compress_generic() will check validity of dstPos and srcPos */
- size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
- *dstPos = output.pos;
- *srcPos = input.pos;
- return cErr;
-}
stage1/zstd/lib/decompress/zstd_decompress_block.c
@@ -1,2072 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-/* zstd_decompress_block :
- * this module takes care of decompressing _compressed_ block */
-
-/*-*******************************************************
-* Dependencies
-*********************************************************/
-#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
-#include "../common/compiler.h" /* prefetch */
-#include "../common/cpu.h" /* bmi2 */
-#include "../common/mem.h" /* low level memory routines */
-#define FSE_STATIC_LINKING_ONLY
-#include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
-#include "../common/huf.h"
-#include "../common/zstd_internal.h"
-#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
-#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
-#include "zstd_decompress_block.h"
-
-/*_*******************************************************
-* Macros
-**********************************************************/
-
-/* These two optional macros force the use one way or another of the two
- * ZSTD_decompressSequences implementations. You can't force in both directions
- * at the same time.
- */
-#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
- defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
-#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!"
-#endif
-
-
-/*_*******************************************************
-* Memory operations
-**********************************************************/
-static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); }
-
-
-/*-*************************************************************
- * Block decoding
- ***************************************************************/
-
-/*! ZSTD_getcBlockSize() :
- * Provides the size of compressed block from block header `src` */
-size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
- blockProperties_t* bpPtr)
-{
- RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, "");
-
- { U32 const cBlockHeader = MEM_readLE24(src);
- U32 const cSize = cBlockHeader >> 3;
- bpPtr->lastBlock = cBlockHeader & 1;
- bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
- bpPtr->origSize = cSize; /* only useful for RLE */
- if (bpPtr->blockType == bt_rle) return 1;
- RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, "");
- return cSize;
- }
-}
-
-/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */
-static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize,
- const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately)
-{
- if (streaming == not_streaming && dstCapacity > ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH)
- {
- /* room for litbuffer to fit without read faulting */
- dctx->litBuffer = (BYTE*)dst + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH;
- dctx->litBufferEnd = dctx->litBuffer + litSize;
- dctx->litBufferLocation = ZSTD_in_dst;
- }
- else if (litSize > ZSTD_LITBUFFEREXTRASIZE)
- {
- /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
- if (splitImmediately) {
- /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
- dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
- dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE;
- }
- else {
- /* initially this will be stored entirely in dst during huffman decoding, it will partially shifted to litExtraBuffer after */
- dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize;
- dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize;
- }
- dctx->litBufferLocation = ZSTD_split;
- }
- else
- {
- /* fits entirely within litExtraBuffer, so no split is necessary */
- dctx->litBuffer = dctx->litExtraBuffer;
- dctx->litBufferEnd = dctx->litBuffer + litSize;
- dctx->litBufferLocation = ZSTD_not_in_dst;
- }
-}
-
-/* Hidden declaration for fullbench */
-size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
- const void* src, size_t srcSize,
- void* dst, size_t dstCapacity, const streaming_operation streaming);
-/*! ZSTD_decodeLiteralsBlock() :
- * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored
- * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current
- * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being
- * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write.
- *
- * @return : nb of bytes read from src (< srcSize )
- * note : symbol not declared but exposed for fullbench */
-size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
- const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */
- void* dst, size_t dstCapacity, const streaming_operation streaming)
-{
- DEBUGLOG(5, "ZSTD_decodeLiteralsBlock");
- RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");
-
- { const BYTE* const istart = (const BYTE*) src;
- symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
-
- switch(litEncType)
- {
- case set_repeat:
- DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block");
- RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, "");
- ZSTD_FALLTHROUGH;
-
- case set_compressed:
- RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3");
- { size_t lhSize, litSize, litCSize;
- U32 singleStream=0;
- U32 const lhlCode = (istart[0] >> 2) & 3;
- U32 const lhc = MEM_readLE32(istart);
- size_t hufSuccess;
- size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
- switch(lhlCode)
- {
- case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
- /* 2 - 2 - 10 - 10 */
- singleStream = !lhlCode;
- lhSize = 3;
- litSize = (lhc >> 4) & 0x3FF;
- litCSize = (lhc >> 14) & 0x3FF;
- break;
- case 2:
- /* 2 - 2 - 14 - 14 */
- lhSize = 4;
- litSize = (lhc >> 4) & 0x3FFF;
- litCSize = lhc >> 18;
- break;
- case 3:
- /* 2 - 2 - 18 - 18 */
- lhSize = 5;
- litSize = (lhc >> 4) & 0x3FFFF;
- litCSize = (lhc >> 22) + ((size_t)istart[4] << 10);
- break;
- }
- RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
- RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
- RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
- RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
- ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
-
- /* prefetch huffman table if cold */
- if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) {
- PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable));
- }
-
- if (litEncType==set_repeat) {
- if (singleStream) {
- hufSuccess = HUF_decompress1X_usingDTable_bmi2(
- dctx->litBuffer, litSize, istart+lhSize, litCSize,
- dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
- } else {
- hufSuccess = HUF_decompress4X_usingDTable_bmi2(
- dctx->litBuffer, litSize, istart+lhSize, litCSize,
- dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
- }
- } else {
- if (singleStream) {
-#if defined(HUF_FORCE_DECOMPRESS_X2)
- hufSuccess = HUF_decompress1X_DCtx_wksp(
- dctx->entropy.hufTable, dctx->litBuffer, litSize,
- istart+lhSize, litCSize, dctx->workspace,
- sizeof(dctx->workspace));
-#else
- hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2(
- dctx->entropy.hufTable, dctx->litBuffer, litSize,
- istart+lhSize, litCSize, dctx->workspace,
- sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
-#endif
- } else {
- hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2(
- dctx->entropy.hufTable, dctx->litBuffer, litSize,
- istart+lhSize, litCSize, dctx->workspace,
- sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
- }
- }
- if (dctx->litBufferLocation == ZSTD_split)
- {
- ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
- ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE);
- dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
- dctx->litBufferEnd -= WILDCOPY_OVERLENGTH;
- }
-
- RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, "");
-
- dctx->litPtr = dctx->litBuffer;
- dctx->litSize = litSize;
- dctx->litEntropy = 1;
- if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
- return litCSize + lhSize;
- }
-
- case set_basic:
- { size_t litSize, lhSize;
- U32 const lhlCode = ((istart[0]) >> 2) & 3;
- size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
- switch(lhlCode)
- {
- case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
- lhSize = 1;
- litSize = istart[0] >> 3;
- break;
- case 1:
- lhSize = 2;
- litSize = MEM_readLE16(istart) >> 4;
- break;
- case 3:
- lhSize = 3;
- litSize = MEM_readLE24(istart) >> 4;
- break;
- }
-
- RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
- RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
- ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
- if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
- RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, "");
- if (dctx->litBufferLocation == ZSTD_split)
- {
- ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE);
- ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
- }
- else
- {
- ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize);
- }
- dctx->litPtr = dctx->litBuffer;
- dctx->litSize = litSize;
- return lhSize+litSize;
- }
- /* direct reference into compressed stream */
- dctx->litPtr = istart+lhSize;
- dctx->litSize = litSize;
- dctx->litBufferEnd = dctx->litPtr + litSize;
- dctx->litBufferLocation = ZSTD_not_in_dst;
- return lhSize+litSize;
- }
-
- case set_rle:
- { U32 const lhlCode = ((istart[0]) >> 2) & 3;
- size_t litSize, lhSize;
- size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
- switch(lhlCode)
- {
- case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
- lhSize = 1;
- litSize = istart[0] >> 3;
- break;
- case 1:
- lhSize = 2;
- litSize = MEM_readLE16(istart) >> 4;
- break;
- case 3:
- lhSize = 3;
- litSize = MEM_readLE24(istart) >> 4;
- RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4");
- break;
- }
- RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
- RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
- RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
- ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
- if (dctx->litBufferLocation == ZSTD_split)
- {
- ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE);
- ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE);
- }
- else
- {
- ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize);
- }
- dctx->litPtr = dctx->litBuffer;
- dctx->litSize = litSize;
- return lhSize+1;
- }
- default:
- RETURN_ERROR(corruption_detected, "impossible");
- }
- }
-}
-
-/* Default FSE distribution tables.
- * These are pre-calculated FSE decoding tables using default distributions as defined in specification :
- * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions
- * They were generated programmatically with following method :
- * - start from default distributions, present in /lib/common/zstd_internal.h
- * - generate tables normally, using ZSTD_buildFSETable()
- * - printout the content of tables
- * - pretify output, report below, test with fuzzer to ensure it's correct */
-
-/* Default FSE distribution table for Literal Lengths */
-static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {
- { 1, 1, 1, LL_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
- /* nextState, nbAddBits, nbBits, baseVal */
- { 0, 0, 4, 0}, { 16, 0, 4, 0},
- { 32, 0, 5, 1}, { 0, 0, 5, 3},
- { 0, 0, 5, 4}, { 0, 0, 5, 6},
- { 0, 0, 5, 7}, { 0, 0, 5, 9},
- { 0, 0, 5, 10}, { 0, 0, 5, 12},
- { 0, 0, 6, 14}, { 0, 1, 5, 16},
- { 0, 1, 5, 20}, { 0, 1, 5, 22},
- { 0, 2, 5, 28}, { 0, 3, 5, 32},
- { 0, 4, 5, 48}, { 32, 6, 5, 64},
- { 0, 7, 5, 128}, { 0, 8, 6, 256},
- { 0, 10, 6, 1024}, { 0, 12, 6, 4096},
- { 32, 0, 4, 0}, { 0, 0, 4, 1},
- { 0, 0, 5, 2}, { 32, 0, 5, 4},
- { 0, 0, 5, 5}, { 32, 0, 5, 7},
- { 0, 0, 5, 8}, { 32, 0, 5, 10},
- { 0, 0, 5, 11}, { 0, 0, 6, 13},
- { 32, 1, 5, 16}, { 0, 1, 5, 18},
- { 32, 1, 5, 22}, { 0, 2, 5, 24},
- { 32, 3, 5, 32}, { 0, 3, 5, 40},
- { 0, 6, 4, 64}, { 16, 6, 4, 64},
- { 32, 7, 5, 128}, { 0, 9, 6, 512},
- { 0, 11, 6, 2048}, { 48, 0, 4, 0},
- { 16, 0, 4, 1}, { 32, 0, 5, 2},
- { 32, 0, 5, 3}, { 32, 0, 5, 5},
- { 32, 0, 5, 6}, { 32, 0, 5, 8},
- { 32, 0, 5, 9}, { 32, 0, 5, 11},
- { 32, 0, 5, 12}, { 0, 0, 6, 15},
- { 32, 1, 5, 18}, { 32, 1, 5, 20},
- { 32, 2, 5, 24}, { 32, 2, 5, 28},
- { 32, 3, 5, 40}, { 32, 4, 5, 48},
- { 0, 16, 6,65536}, { 0, 15, 6,32768},
- { 0, 14, 6,16384}, { 0, 13, 6, 8192},
-}; /* LL_defaultDTable */
-
-/* Default FSE distribution table for Offset Codes */
-static const ZSTD_seqSymbol OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {
- { 1, 1, 1, OF_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
- /* nextState, nbAddBits, nbBits, baseVal */
- { 0, 0, 5, 0}, { 0, 6, 4, 61},
- { 0, 9, 5, 509}, { 0, 15, 5,32765},
- { 0, 21, 5,2097149}, { 0, 3, 5, 5},
- { 0, 7, 4, 125}, { 0, 12, 5, 4093},
- { 0, 18, 5,262141}, { 0, 23, 5,8388605},
- { 0, 5, 5, 29}, { 0, 8, 4, 253},
- { 0, 14, 5,16381}, { 0, 20, 5,1048573},
- { 0, 2, 5, 1}, { 16, 7, 4, 125},
- { 0, 11, 5, 2045}, { 0, 17, 5,131069},
- { 0, 22, 5,4194301}, { 0, 4, 5, 13},
- { 16, 8, 4, 253}, { 0, 13, 5, 8189},
- { 0, 19, 5,524285}, { 0, 1, 5, 1},
- { 16, 6, 4, 61}, { 0, 10, 5, 1021},
- { 0, 16, 5,65533}, { 0, 28, 5,268435453},
- { 0, 27, 5,134217725}, { 0, 26, 5,67108861},
- { 0, 25, 5,33554429}, { 0, 24, 5,16777213},
-}; /* OF_defaultDTable */
-
-
-/* Default FSE distribution table for Match Lengths */
-static const ZSTD_seqSymbol ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = {
- { 1, 1, 1, ML_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
- /* nextState, nbAddBits, nbBits, baseVal */
- { 0, 0, 6, 3}, { 0, 0, 4, 4},
- { 32, 0, 5, 5}, { 0, 0, 5, 6},
- { 0, 0, 5, 8}, { 0, 0, 5, 9},
- { 0, 0, 5, 11}, { 0, 0, 6, 13},
- { 0, 0, 6, 16}, { 0, 0, 6, 19},
- { 0, 0, 6, 22}, { 0, 0, 6, 25},
- { 0, 0, 6, 28}, { 0, 0, 6, 31},
- { 0, 0, 6, 34}, { 0, 1, 6, 37},
- { 0, 1, 6, 41}, { 0, 2, 6, 47},
- { 0, 3, 6, 59}, { 0, 4, 6, 83},
- { 0, 7, 6, 131}, { 0, 9, 6, 515},
- { 16, 0, 4, 4}, { 0, 0, 4, 5},
- { 32, 0, 5, 6}, { 0, 0, 5, 7},
- { 32, 0, 5, 9}, { 0, 0, 5, 10},
- { 0, 0, 6, 12}, { 0, 0, 6, 15},
- { 0, 0, 6, 18}, { 0, 0, 6, 21},
- { 0, 0, 6, 24}, { 0, 0, 6, 27},
- { 0, 0, 6, 30}, { 0, 0, 6, 33},
- { 0, 1, 6, 35}, { 0, 1, 6, 39},
- { 0, 2, 6, 43}, { 0, 3, 6, 51},
- { 0, 4, 6, 67}, { 0, 5, 6, 99},
- { 0, 8, 6, 259}, { 32, 0, 4, 4},
- { 48, 0, 4, 4}, { 16, 0, 4, 5},
- { 32, 0, 5, 7}, { 32, 0, 5, 8},
- { 32, 0, 5, 10}, { 32, 0, 5, 11},
- { 0, 0, 6, 14}, { 0, 0, 6, 17},
- { 0, 0, 6, 20}, { 0, 0, 6, 23},
- { 0, 0, 6, 26}, { 0, 0, 6, 29},
- { 0, 0, 6, 32}, { 0, 16, 6,65539},
- { 0, 15, 6,32771}, { 0, 14, 6,16387},
- { 0, 13, 6, 8195}, { 0, 12, 6, 4099},
- { 0, 11, 6, 2051}, { 0, 10, 6, 1027},
-}; /* ML_defaultDTable */
-
-
-static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U8 nbAddBits)
-{
- void* ptr = dt;
- ZSTD_seqSymbol_header* const DTableH = (ZSTD_seqSymbol_header*)ptr;
- ZSTD_seqSymbol* const cell = dt + 1;
-
- DTableH->tableLog = 0;
- DTableH->fastMode = 0;
-
- cell->nbBits = 0;
- cell->nextState = 0;
- assert(nbAddBits < 255);
- cell->nbAdditionalBits = nbAddBits;
- cell->baseValue = baseValue;
-}
-
-
-/* ZSTD_buildFSETable() :
- * generate FSE decoding table for one symbol (ll, ml or off)
- * cannot fail if input is valid =>
- * all inputs are presumed validated at this stage */
-FORCE_INLINE_TEMPLATE
-void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
- const short* normalizedCounter, unsigned maxSymbolValue,
- const U32* baseValue, const U8* nbAdditionalBits,
- unsigned tableLog, void* wksp, size_t wkspSize)
-{
- ZSTD_seqSymbol* const tableDecode = dt+1;
- U32 const maxSV1 = maxSymbolValue + 1;
- U32 const tableSize = 1 << tableLog;
-
- U16* symbolNext = (U16*)wksp;
- BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1);
- U32 highThreshold = tableSize - 1;
-
-
- /* Sanity Checks */
- assert(maxSymbolValue <= MaxSeq);
- assert(tableLog <= MaxFSELog);
- assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE);
- (void)wkspSize;
- /* Init, lay down lowprob symbols */
- { ZSTD_seqSymbol_header DTableH;
- DTableH.tableLog = tableLog;
- DTableH.fastMode = 1;
- { S16 const largeLimit= (S16)(1 << (tableLog-1));
- U32 s;
- for (s=0; s<maxSV1; s++) {
- if (normalizedCounter[s]==-1) {
- tableDecode[highThreshold--].baseValue = s;
- symbolNext[s] = 1;
- } else {
- if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
- assert(normalizedCounter[s]>=0);
- symbolNext[s] = (U16)normalizedCounter[s];
- } } }
- ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
- }
-
- /* Spread symbols */
- assert(tableSize <= 512);
- /* Specialized symbol spreading for the case when there are
- * no low probability (-1 count) symbols. When compressing
- * small blocks we avoid low probability symbols to hit this
- * case, since header decoding speed matters more.
- */
- if (highThreshold == tableSize - 1) {
- size_t const tableMask = tableSize-1;
- size_t const step = FSE_TABLESTEP(tableSize);
- /* First lay down the symbols in order.
- * We use a uint64_t to lay down 8 bytes at a time. This reduces branch
- * misses since small blocks generally have small table logs, so nearly
- * all symbols have counts <= 8. We ensure we have 8 bytes at the end of
- * our buffer to handle the over-write.
- */
- {
- U64 const add = 0x0101010101010101ull;
- size_t pos = 0;
- U64 sv = 0;
- U32 s;
- for (s=0; s<maxSV1; ++s, sv += add) {
- int i;
- int const n = normalizedCounter[s];
- MEM_write64(spread + pos, sv);
- for (i = 8; i < n; i += 8) {
- MEM_write64(spread + pos + i, sv);
- }
- pos += n;
- }
- }
- /* Now we spread those positions across the table.
- * The benefit of doing it in two stages is that we avoid the the
- * variable size inner loop, which caused lots of branch misses.
- * Now we can run through all the positions without any branch misses.
- * We unroll the loop twice, since that is what emperically worked best.
- */
- {
- size_t position = 0;
- size_t s;
- size_t const unroll = 2;
- assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
- for (s = 0; s < (size_t)tableSize; s += unroll) {
- size_t u;
- for (u = 0; u < unroll; ++u) {
- size_t const uPosition = (position + (u * step)) & tableMask;
- tableDecode[uPosition].baseValue = spread[s + u];
- }
- position = (position + (unroll * step)) & tableMask;
- }
- assert(position == 0);
- }
- } else {
- U32 const tableMask = tableSize-1;
- U32 const step = FSE_TABLESTEP(tableSize);
- U32 s, position = 0;
- for (s=0; s<maxSV1; s++) {
- int i;
- int const n = normalizedCounter[s];
- for (i=0; i<n; i++) {
- tableDecode[position].baseValue = s;
- position = (position + step) & tableMask;
- while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
- } }
- assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
- }
-
- /* Build Decoding table */
- {
- U32 u;
- for (u=0; u<tableSize; u++) {
- U32 const symbol = tableDecode[u].baseValue;
- U32 const nextState = symbolNext[symbol]++;
- tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
- tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
- assert(nbAdditionalBits[symbol] < 255);
- tableDecode[u].nbAdditionalBits = nbAdditionalBits[symbol];
- tableDecode[u].baseValue = baseValue[symbol];
- }
- }
-}
-
-/* Avoids the FORCE_INLINE of the _body() function. */
-static void ZSTD_buildFSETable_body_default(ZSTD_seqSymbol* dt,
- const short* normalizedCounter, unsigned maxSymbolValue,
- const U32* baseValue, const U8* nbAdditionalBits,
- unsigned tableLog, void* wksp, size_t wkspSize)
-{
- ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
- baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
-}
-
-#if DYNAMIC_BMI2
-BMI2_TARGET_ATTRIBUTE static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* dt,
- const short* normalizedCounter, unsigned maxSymbolValue,
- const U32* baseValue, const U8* nbAdditionalBits,
- unsigned tableLog, void* wksp, size_t wkspSize)
-{
- ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
- baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
-}
-#endif
-
-void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
- const short* normalizedCounter, unsigned maxSymbolValue,
- const U32* baseValue, const U8* nbAdditionalBits,
- unsigned tableLog, void* wksp, size_t wkspSize, int bmi2)
-{
-#if DYNAMIC_BMI2
- if (bmi2) {
- ZSTD_buildFSETable_body_bmi2(dt, normalizedCounter, maxSymbolValue,
- baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
- return;
- }
-#endif
- (void)bmi2;
- ZSTD_buildFSETable_body_default(dt, normalizedCounter, maxSymbolValue,
- baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
-}
-
-
-/*! ZSTD_buildSeqTable() :
- * @return : nb bytes read from src,
- * or an error code if it fails */
-static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr,
- symbolEncodingType_e type, unsigned max, U32 maxLog,
- const void* src, size_t srcSize,
- const U32* baseValue, const U8* nbAdditionalBits,
- const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable,
- int ddictIsCold, int nbSeq, U32* wksp, size_t wkspSize,
- int bmi2)
-{
- switch(type)
- {
- case set_rle :
- RETURN_ERROR_IF(!srcSize, srcSize_wrong, "");
- RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected, "");
- { U32 const symbol = *(const BYTE*)src;
- U32 const baseline = baseValue[symbol];
- U8 const nbBits = nbAdditionalBits[symbol];
- ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits);
- }
- *DTablePtr = DTableSpace;
- return 1;
- case set_basic :
- *DTablePtr = defaultTable;
- return 0;
- case set_repeat:
- RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, "");
- /* prefetch FSE table if used */
- if (ddictIsCold && (nbSeq > 24 /* heuristic */)) {
- const void* const pStart = *DTablePtr;
- size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog));
- PREFETCH_AREA(pStart, pSize);
- }
- return 0;
- case set_compressed :
- { unsigned tableLog;
- S16 norm[MaxSeq+1];
- size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
- RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, "");
- RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, "");
- ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2);
- *DTablePtr = DTableSpace;
- return headerSize;
- }
- default :
- assert(0);
- RETURN_ERROR(GENERIC, "impossible");
- }
-}
-
-size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
- const void* src, size_t srcSize)
-{
- const BYTE* const istart = (const BYTE*)src;
- const BYTE* const iend = istart + srcSize;
- const BYTE* ip = istart;
- int nbSeq;
- DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
-
- /* check */
- RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, "");
-
- /* SeqHead */
- nbSeq = *ip++;
- if (!nbSeq) {
- *nbSeqPtr=0;
- RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, "");
- return 1;
- }
- if (nbSeq > 0x7F) {
- if (nbSeq == 0xFF) {
- RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, "");
- nbSeq = MEM_readLE16(ip) + LONGNBSEQ;
- ip+=2;
- } else {
- RETURN_ERROR_IF(ip >= iend, srcSize_wrong, "");
- nbSeq = ((nbSeq-0x80)<<8) + *ip++;
- }
- }
- *nbSeqPtr = nbSeq;
-
- /* FSE table descriptors */
- RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */
- { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
- symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
- symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
- ip++;
-
- /* Build DTables */
- { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
- LLtype, MaxLL, LLFSELog,
- ip, iend-ip,
- LL_base, LL_bits,
- LL_defaultDTable, dctx->fseEntropy,
- dctx->ddictIsCold, nbSeq,
- dctx->workspace, sizeof(dctx->workspace),
- ZSTD_DCtx_get_bmi2(dctx));
- RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed");
- ip += llhSize;
- }
-
- { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
- OFtype, MaxOff, OffFSELog,
- ip, iend-ip,
- OF_base, OF_bits,
- OF_defaultDTable, dctx->fseEntropy,
- dctx->ddictIsCold, nbSeq,
- dctx->workspace, sizeof(dctx->workspace),
- ZSTD_DCtx_get_bmi2(dctx));
- RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed");
- ip += ofhSize;
- }
-
- { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
- MLtype, MaxML, MLFSELog,
- ip, iend-ip,
- ML_base, ML_bits,
- ML_defaultDTable, dctx->fseEntropy,
- dctx->ddictIsCold, nbSeq,
- dctx->workspace, sizeof(dctx->workspace),
- ZSTD_DCtx_get_bmi2(dctx));
- RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed");
- ip += mlhSize;
- }
- }
-
- return ip-istart;
-}
-
-
-typedef struct {
- size_t litLength;
- size_t matchLength;
- size_t offset;
-} seq_t;
-
-typedef struct {
- size_t state;
- const ZSTD_seqSymbol* table;
-} ZSTD_fseState;
-
-typedef struct {
- BIT_DStream_t DStream;
- ZSTD_fseState stateLL;
- ZSTD_fseState stateOffb;
- ZSTD_fseState stateML;
- size_t prevOffset[ZSTD_REP_NUM];
-} seqState_t;
-
-/*! ZSTD_overlapCopy8() :
- * Copies 8 bytes from ip to op and updates op and ip where ip <= op.
- * If the offset is < 8 then the offset is spread to at least 8 bytes.
- *
- * Precondition: *ip <= *op
- * Postcondition: *op - *op >= 8
- */
-HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) {
- assert(*ip <= *op);
- if (offset < 8) {
- /* close range match, overlap */
- static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
- int const sub2 = dec64table[offset];
- (*op)[0] = (*ip)[0];
- (*op)[1] = (*ip)[1];
- (*op)[2] = (*ip)[2];
- (*op)[3] = (*ip)[3];
- *ip += dec32table[offset];
- ZSTD_copy4(*op+4, *ip);
- *ip -= sub2;
- } else {
- ZSTD_copy8(*op, *ip);
- }
- *ip += 8;
- *op += 8;
- assert(*op - *ip >= 8);
-}
-
-/*! ZSTD_safecopy() :
- * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer
- * and write up to 16 bytes past oend_w (op >= oend_w is allowed).
- * This function is only called in the uncommon case where the sequence is near the end of the block. It
- * should be fast for a single long sequence, but can be slow for several short sequences.
- *
- * @param ovtype controls the overlap detection
- * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
- * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
- * The src buffer must be before the dst buffer.
- */
-static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) {
- ptrdiff_t const diff = op - ip;
- BYTE* const oend = op + length;
-
- assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) ||
- (ovtype == ZSTD_overlap_src_before_dst && diff >= 0));
-
- if (length < 8) {
- /* Handle short lengths. */
- while (op < oend) *op++ = *ip++;
- return;
- }
- if (ovtype == ZSTD_overlap_src_before_dst) {
- /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */
- assert(length >= 8);
- ZSTD_overlapCopy8(&op, &ip, diff);
- length -= 8;
- assert(op - ip >= 8);
- assert(op <= oend);
- }
-
- if (oend <= oend_w) {
- /* No risk of overwrite. */
- ZSTD_wildcopy(op, ip, length, ovtype);
- return;
- }
- if (op <= oend_w) {
- /* Wildcopy until we get close to the end. */
- assert(oend > oend_w);
- ZSTD_wildcopy(op, ip, oend_w - op, ovtype);
- ip += oend_w - op;
- op += oend_w - op;
- }
- /* Handle the leftovers. */
- while (op < oend) *op++ = *ip++;
-}
-
-/* ZSTD_safecopyDstBeforeSrc():
- * This version allows overlap with dst before src, or handles the non-overlap case with dst after src
- * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */
-static void ZSTD_safecopyDstBeforeSrc(BYTE* op, BYTE const* ip, ptrdiff_t length) {
- ptrdiff_t const diff = op - ip;
- BYTE* const oend = op + length;
-
- if (length < 8 || diff > -8) {
- /* Handle short lengths, close overlaps, and dst not before src. */
- while (op < oend) *op++ = *ip++;
- return;
- }
-
- if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) {
- ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap);
- ip += oend - WILDCOPY_OVERLENGTH - op;
- op += oend - WILDCOPY_OVERLENGTH - op;
- }
-
- /* Handle the leftovers. */
- while (op < oend) *op++ = *ip++;
-}
-
-/* ZSTD_execSequenceEnd():
- * This version handles cases that are near the end of the output buffer. It requires
- * more careful checks to make sure there is no overflow. By separating out these hard
- * and unlikely cases, we can speed up the common cases.
- *
- * NOTE: This function needs to be fast for a single long sequence, but doesn't need
- * to be optimized for many small sequences, since those fall into ZSTD_execSequence().
- */
-FORCE_NOINLINE
-size_t ZSTD_execSequenceEnd(BYTE* op,
- BYTE* const oend, seq_t sequence,
- const BYTE** litPtr, const BYTE* const litLimit,
- const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
-{
- BYTE* const oLitEnd = op + sequence.litLength;
- size_t const sequenceLength = sequence.litLength + sequence.matchLength;
- const BYTE* const iLitEnd = *litPtr + sequence.litLength;
- const BYTE* match = oLitEnd - sequence.offset;
- BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
-
- /* bounds checks : careful of address space overflow in 32-bit mode */
- RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
- RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
- assert(op < op + sequenceLength);
- assert(oLitEnd < op + sequenceLength);
-
- /* copy literals */
- ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap);
- op = oLitEnd;
- *litPtr = iLitEnd;
-
- /* copy Match */
- if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
- /* offset beyond prefix */
- RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
- match = dictEnd - (prefixStart - match);
- if (match + sequence.matchLength <= dictEnd) {
- ZSTD_memmove(oLitEnd, match, sequence.matchLength);
- return sequenceLength;
- }
- /* span extDict & currentPrefixSegment */
- { size_t const length1 = dictEnd - match;
- ZSTD_memmove(oLitEnd, match, length1);
- op = oLitEnd + length1;
- sequence.matchLength -= length1;
- match = prefixStart;
- }
- }
- ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
- return sequenceLength;
-}
-
-/* ZSTD_execSequenceEndSplitLitBuffer():
- * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case.
- */
-FORCE_NOINLINE
-size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op,
- BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
- const BYTE** litPtr, const BYTE* const litLimit,
- const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
-{
- BYTE* const oLitEnd = op + sequence.litLength;
- size_t const sequenceLength = sequence.litLength + sequence.matchLength;
- const BYTE* const iLitEnd = *litPtr + sequence.litLength;
- const BYTE* match = oLitEnd - sequence.offset;
-
-
- /* bounds checks : careful of address space overflow in 32-bit mode */
- RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
- RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
- assert(op < op + sequenceLength);
- assert(oLitEnd < op + sequenceLength);
-
- /* copy literals */
- RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer");
- ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength);
- op = oLitEnd;
- *litPtr = iLitEnd;
-
- /* copy Match */
- if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
- /* offset beyond prefix */
- RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
- match = dictEnd - (prefixStart - match);
- if (match + sequence.matchLength <= dictEnd) {
- ZSTD_memmove(oLitEnd, match, sequence.matchLength);
- return sequenceLength;
- }
- /* span extDict & currentPrefixSegment */
- { size_t const length1 = dictEnd - match;
- ZSTD_memmove(oLitEnd, match, length1);
- op = oLitEnd + length1;
- sequence.matchLength -= length1;
- match = prefixStart;
- }
- }
- ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
- return sequenceLength;
-}
-
-HINT_INLINE
-size_t ZSTD_execSequence(BYTE* op,
- BYTE* const oend, seq_t sequence,
- const BYTE** litPtr, const BYTE* const litLimit,
- const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
-{
- BYTE* const oLitEnd = op + sequence.litLength;
- size_t const sequenceLength = sequence.litLength + sequence.matchLength;
- BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
- BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */
- const BYTE* const iLitEnd = *litPtr + sequence.litLength;
- const BYTE* match = oLitEnd - sequence.offset;
-
- assert(op != NULL /* Precondition */);
- assert(oend_w < oend /* No underflow */);
- /* Handle edge cases in a slow path:
- * - Read beyond end of literals
- * - Match end is within WILDCOPY_OVERLIMIT of oend
- * - 32-bit mode and the match length overflows
- */
- if (UNLIKELY(
- iLitEnd > litLimit ||
- oMatchEnd > oend_w ||
- (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
- return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
-
- /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
- assert(op <= oLitEnd /* No overflow */);
- assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
- assert(oMatchEnd <= oend /* No underflow */);
- assert(iLitEnd <= litLimit /* Literal length is in bounds */);
- assert(oLitEnd <= oend_w /* Can wildcopy literals */);
- assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
-
- /* Copy Literals:
- * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
- * We likely don't need the full 32-byte wildcopy.
- */
- assert(WILDCOPY_OVERLENGTH >= 16);
- ZSTD_copy16(op, (*litPtr));
- if (UNLIKELY(sequence.litLength > 16)) {
- ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap);
- }
- op = oLitEnd;
- *litPtr = iLitEnd; /* update for next sequence */
-
- /* Copy Match */
- if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
- /* offset beyond prefix -> go into extDict */
- RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
- match = dictEnd + (match - prefixStart);
- if (match + sequence.matchLength <= dictEnd) {
- ZSTD_memmove(oLitEnd, match, sequence.matchLength);
- return sequenceLength;
- }
- /* span extDict & currentPrefixSegment */
- { size_t const length1 = dictEnd - match;
- ZSTD_memmove(oLitEnd, match, length1);
- op = oLitEnd + length1;
- sequence.matchLength -= length1;
- match = prefixStart;
- }
- }
- /* Match within prefix of 1 or more bytes */
- assert(op <= oMatchEnd);
- assert(oMatchEnd <= oend_w);
- assert(match >= prefixStart);
- assert(sequence.matchLength >= 1);
-
- /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
- * without overlap checking.
- */
- if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
- /* We bet on a full wildcopy for matches, since we expect matches to be
- * longer than literals (in general). In silesia, ~10% of matches are longer
- * than 16 bytes.
- */
- ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
- return sequenceLength;
- }
- assert(sequence.offset < WILDCOPY_VECLEN);
-
- /* Copy 8 bytes and spread the offset to be >= 8. */
- ZSTD_overlapCopy8(&op, &match, sequence.offset);
-
- /* If the match length is > 8 bytes, then continue with the wildcopy. */
- if (sequence.matchLength > 8) {
- assert(op < oMatchEnd);
- ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst);
- }
- return sequenceLength;
-}
-
-HINT_INLINE
-size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op,
- BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
- const BYTE** litPtr, const BYTE* const litLimit,
- const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
-{
- BYTE* const oLitEnd = op + sequence.litLength;
- size_t const sequenceLength = sequence.litLength + sequence.matchLength;
- BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
- const BYTE* const iLitEnd = *litPtr + sequence.litLength;
- const BYTE* match = oLitEnd - sequence.offset;
-
- assert(op != NULL /* Precondition */);
- assert(oend_w < oend /* No underflow */);
- /* Handle edge cases in a slow path:
- * - Read beyond end of literals
- * - Match end is within WILDCOPY_OVERLIMIT of oend
- * - 32-bit mode and the match length overflows
- */
- if (UNLIKELY(
- iLitEnd > litLimit ||
- oMatchEnd > oend_w ||
- (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
- return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
-
- /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
- assert(op <= oLitEnd /* No overflow */);
- assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
- assert(oMatchEnd <= oend /* No underflow */);
- assert(iLitEnd <= litLimit /* Literal length is in bounds */);
- assert(oLitEnd <= oend_w /* Can wildcopy literals */);
- assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
-
- /* Copy Literals:
- * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
- * We likely don't need the full 32-byte wildcopy.
- */
- assert(WILDCOPY_OVERLENGTH >= 16);
- ZSTD_copy16(op, (*litPtr));
- if (UNLIKELY(sequence.litLength > 16)) {
- ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap);
- }
- op = oLitEnd;
- *litPtr = iLitEnd; /* update for next sequence */
-
- /* Copy Match */
- if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
- /* offset beyond prefix -> go into extDict */
- RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
- match = dictEnd + (match - prefixStart);
- if (match + sequence.matchLength <= dictEnd) {
- ZSTD_memmove(oLitEnd, match, sequence.matchLength);
- return sequenceLength;
- }
- /* span extDict & currentPrefixSegment */
- { size_t const length1 = dictEnd - match;
- ZSTD_memmove(oLitEnd, match, length1);
- op = oLitEnd + length1;
- sequence.matchLength -= length1;
- match = prefixStart;
- } }
- /* Match within prefix of 1 or more bytes */
- assert(op <= oMatchEnd);
- assert(oMatchEnd <= oend_w);
- assert(match >= prefixStart);
- assert(sequence.matchLength >= 1);
-
- /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
- * without overlap checking.
- */
- if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
- /* We bet on a full wildcopy for matches, since we expect matches to be
- * longer than literals (in general). In silesia, ~10% of matches are longer
- * than 16 bytes.
- */
- ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
- return sequenceLength;
- }
- assert(sequence.offset < WILDCOPY_VECLEN);
-
- /* Copy 8 bytes and spread the offset to be >= 8. */
- ZSTD_overlapCopy8(&op, &match, sequence.offset);
-
- /* If the match length is > 8 bytes, then continue with the wildcopy. */
- if (sequence.matchLength > 8) {
- assert(op < oMatchEnd);
- ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst);
- }
- return sequenceLength;
-}
-
-
-static void
-ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt)
-{
- const void* ptr = dt;
- const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr;
- DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
- DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits",
- (U32)DStatePtr->state, DTableH->tableLog);
- BIT_reloadDStream(bitD);
- DStatePtr->table = dt + 1;
-}
-
-FORCE_INLINE_TEMPLATE void
-ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits)
-{
- size_t const lowBits = BIT_readBits(bitD, nbBits);
- DStatePtr->state = nextState + lowBits;
-}
-
-/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
- * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1)
- * bits before reloading. This value is the maximum number of bytes we read
- * after reloading when we are decoding long offsets.
- */
-#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \
- (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \
- ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \
- : 0)
-
-typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e;
-
-FORCE_INLINE_TEMPLATE seq_t
-ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
-{
- seq_t seq;
- const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
- const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
- const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
- seq.matchLength = mlDInfo->baseValue;
- seq.litLength = llDInfo->baseValue;
- { U32 const ofBase = ofDInfo->baseValue;
- BYTE const llBits = llDInfo->nbAdditionalBits;
- BYTE const mlBits = mlDInfo->nbAdditionalBits;
- BYTE const ofBits = ofDInfo->nbAdditionalBits;
- BYTE const totalBits = llBits+mlBits+ofBits;
-
- U16 const llNext = llDInfo->nextState;
- U16 const mlNext = mlDInfo->nextState;
- U16 const ofNext = ofDInfo->nextState;
- U32 const llnbBits = llDInfo->nbBits;
- U32 const mlnbBits = mlDInfo->nbBits;
- U32 const ofnbBits = ofDInfo->nbBits;
- /*
- * As gcc has better branch and block analyzers, sometimes it is only
- * valuable to mark likelyness for clang, it gives around 3-4% of
- * performance.
- */
-
- /* sequence */
- { size_t offset;
- #if defined(__clang__)
- if (LIKELY(ofBits > 1)) {
- #else
- if (ofBits > 1) {
- #endif
- ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
- ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
- assert(ofBits <= MaxOff);
- if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) {
- U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed);
- offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
- BIT_reloadDStream(&seqState->DStream);
- if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
- assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */
- } else {
- offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
- if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
- }
- seqState->prevOffset[2] = seqState->prevOffset[1];
- seqState->prevOffset[1] = seqState->prevOffset[0];
- seqState->prevOffset[0] = offset;
- } else {
- U32 const ll0 = (llDInfo->baseValue == 0);
- if (LIKELY((ofBits == 0))) {
- offset = seqState->prevOffset[ll0];
- seqState->prevOffset[1] = seqState->prevOffset[!ll0];
- seqState->prevOffset[0] = offset;
- } else {
- offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1);
- { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
- temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
- if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
- seqState->prevOffset[1] = seqState->prevOffset[0];
- seqState->prevOffset[0] = offset = temp;
- } } }
- seq.offset = offset;
- }
-
- #if defined(__clang__)
- if (UNLIKELY(mlBits > 0))
- #else
- if (mlBits > 0)
- #endif
- seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/);
-
- if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
- BIT_reloadDStream(&seqState->DStream);
- if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
- BIT_reloadDStream(&seqState->DStream);
- /* Ensure there are enough bits to read the rest of data in 64-bit mode. */
- ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
-
- #if defined(__clang__)
- if (UNLIKELY(llBits > 0))
- #else
- if (llBits > 0)
- #endif
- seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
-
- if (MEM_32bits())
- BIT_reloadDStream(&seqState->DStream);
-
- DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
- (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
-
- ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */
- ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */
- if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
- ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */
- }
-
- return seq;
-}
-
-#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
-MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd)
-{
- size_t const windowSize = dctx->fParams.windowSize;
- /* No dictionary used. */
- if (dctx->dictContentEndForFuzzing == NULL) return 0;
- /* Dictionary is our prefix. */
- if (prefixStart == dctx->dictContentBeginForFuzzing) return 1;
- /* Dictionary is not our ext-dict. */
- if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0;
- /* Dictionary is not within our window size. */
- if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0;
- /* Dictionary is active. */
- return 1;
-}
-
-MEM_STATIC void ZSTD_assertValidSequence(
- ZSTD_DCtx const* dctx,
- BYTE const* op, BYTE const* oend,
- seq_t const seq,
- BYTE const* prefixStart, BYTE const* virtualStart)
-{
-#if DEBUGLEVEL >= 1
- size_t const windowSize = dctx->fParams.windowSize;
- size_t const sequenceSize = seq.litLength + seq.matchLength;
- BYTE const* const oLitEnd = op + seq.litLength;
- DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u",
- (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
- assert(op <= oend);
- assert((size_t)(oend - op) >= sequenceSize);
- assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX);
- if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) {
- size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing);
- /* Offset must be within the dictionary. */
- assert(seq.offset <= (size_t)(oLitEnd - virtualStart));
- assert(seq.offset <= windowSize + dictSize);
- } else {
- /* Offset must be within our window. */
- assert(seq.offset <= windowSize);
- }
-#else
- (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart;
-#endif
-}
-#endif
-
-#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
-
-
-FORCE_INLINE_TEMPLATE size_t
-DONT_VECTORIZE
-ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- const BYTE* ip = (const BYTE*)seqStart;
- const BYTE* const iend = ip + seqSize;
- BYTE* const ostart = (BYTE*)dst;
- BYTE* const oend = ostart + maxDstSize;
- BYTE* op = ostart;
- const BYTE* litPtr = dctx->litPtr;
- const BYTE* litBufferEnd = dctx->litBufferEnd;
- const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
- const BYTE* const vBase = (const BYTE*) (dctx->virtualStart);
- const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
- DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer");
- (void)frame;
-
- /* Regen sequences */
- if (nbSeq) {
- seqState_t seqState;
- dctx->fseEntropy = 1;
- { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
- RETURN_ERROR_IF(
- ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
- corruption_detected, "");
- ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
- ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
- ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
- assert(dst != NULL);
-
- ZSTD_STATIC_ASSERT(
- BIT_DStream_unfinished < BIT_DStream_completed &&
- BIT_DStream_endOfBuffer < BIT_DStream_completed &&
- BIT_DStream_completed < BIT_DStream_overflow);
-
- /* decompress without overrunning litPtr begins */
- {
- seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
- /* Align the decompression loop to 32 + 16 bytes.
- *
- * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression
- * speed swings based on the alignment of the decompression loop. This
- * performance swing is caused by parts of the decompression loop falling
- * out of the DSB. The entire decompression loop should fit in the DSB,
- * when it can't we get much worse performance. You can measure if you've
- * hit the good case or the bad case with this perf command for some
- * compressed file test.zst:
- *
- * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \
- * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst
- *
- * If you see most cycles served out of the MITE you've hit the bad case.
- * If you see most cycles served out of the DSB you've hit the good case.
- * If it is pretty even then you may be in an okay case.
- *
- * This issue has been reproduced on the following CPUs:
- * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9
- * Use Instruments->Counters to get DSB/MITE cycles.
- * I never got performance swings, but I was able to
- * go from the good case of mostly DSB to half of the
- * cycles served from MITE.
- * - Coffeelake: Intel i9-9900k
- * - Coffeelake: Intel i7-9700k
- *
- * I haven't been able to reproduce the instability or DSB misses on any
- * of the following CPUS:
- * - Haswell
- * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH
- * - Skylake
- *
- * Alignment is done for each of the three major decompression loops:
- * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer
- * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer
- * - ZSTD_decompressSequences_body
- * Alignment choices are made to minimize large swings on bad cases and influence on performance
- * from changes external to this code, rather than to overoptimize on the current commit.
- *
- * If you are seeing performance stability this script can help test.
- * It tests on 4 commits in zstd where I saw performance change.
- *
- * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4
- */
-#if defined(__GNUC__) && defined(__x86_64__)
- __asm__(".p2align 6");
-# if __GNUC__ >= 7
- /* good for gcc-7, gcc-9, and gcc-11 */
- __asm__("nop");
- __asm__(".p2align 5");
- __asm__("nop");
- __asm__(".p2align 4");
-# if __GNUC__ == 8 || __GNUC__ == 10
- /* good for gcc-8 and gcc-10 */
- __asm__("nop");
- __asm__(".p2align 3");
-# endif
-# endif
-#endif
-
- /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */
- for (; litPtr + sequence.litLength <= dctx->litBufferEnd; ) {
- size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
-#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
- assert(!ZSTD_isError(oneSeqSize));
- if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
-#endif
- if (UNLIKELY(ZSTD_isError(oneSeqSize)))
- return oneSeqSize;
- DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
- op += oneSeqSize;
- if (UNLIKELY(!--nbSeq))
- break;
- BIT_reloadDStream(&(seqState.DStream));
- sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
- }
-
- /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */
- if (nbSeq > 0) {
- const size_t leftoverLit = dctx->litBufferEnd - litPtr;
- if (leftoverLit)
- {
- RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
- ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
- sequence.litLength -= leftoverLit;
- op += leftoverLit;
- }
- litPtr = dctx->litExtraBuffer;
- litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
- dctx->litBufferLocation = ZSTD_not_in_dst;
- {
- size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
-#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
- assert(!ZSTD_isError(oneSeqSize));
- if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
-#endif
- if (UNLIKELY(ZSTD_isError(oneSeqSize)))
- return oneSeqSize;
- DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
- op += oneSeqSize;
- if (--nbSeq)
- BIT_reloadDStream(&(seqState.DStream));
- }
- }
- }
-
- if (nbSeq > 0) /* there is remaining lit from extra buffer */
- {
-
-#if defined(__GNUC__) && defined(__x86_64__)
- __asm__(".p2align 6");
- __asm__("nop");
-# if __GNUC__ != 7
- /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */
- __asm__(".p2align 4");
- __asm__("nop");
- __asm__(".p2align 3");
-# elif __GNUC__ >= 11
- __asm__(".p2align 3");
-# else
- __asm__(".p2align 5");
- __asm__("nop");
- __asm__(".p2align 3");
-# endif
-#endif
-
- for (; ; ) {
- seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
- size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
-#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
- assert(!ZSTD_isError(oneSeqSize));
- if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
-#endif
- if (UNLIKELY(ZSTD_isError(oneSeqSize)))
- return oneSeqSize;
- DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
- op += oneSeqSize;
- if (UNLIKELY(!--nbSeq))
- break;
- BIT_reloadDStream(&(seqState.DStream));
- }
- }
-
- /* check if reached exact end */
- DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq);
- RETURN_ERROR_IF(nbSeq, corruption_detected, "");
- RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, "");
- /* save reps for next block */
- { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
- }
-
- /* last literal segment */
- if (dctx->litBufferLocation == ZSTD_split) /* split hasn't been reached yet, first get dst then copy litExtraBuffer */
- {
- size_t const lastLLSize = litBufferEnd - litPtr;
- RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
- if (op != NULL) {
- ZSTD_memmove(op, litPtr, lastLLSize);
- op += lastLLSize;
- }
- litPtr = dctx->litExtraBuffer;
- litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
- dctx->litBufferLocation = ZSTD_not_in_dst;
- }
- { size_t const lastLLSize = litBufferEnd - litPtr;
- RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
- if (op != NULL) {
- ZSTD_memcpy(op, litPtr, lastLLSize);
- op += lastLLSize;
- }
- }
-
- return op-ostart;
-}
-
-FORCE_INLINE_TEMPLATE size_t
-DONT_VECTORIZE
-ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- const BYTE* ip = (const BYTE*)seqStart;
- const BYTE* const iend = ip + seqSize;
- BYTE* const ostart = (BYTE*)dst;
- BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ostart + maxDstSize : dctx->litBuffer;
- BYTE* op = ostart;
- const BYTE* litPtr = dctx->litPtr;
- const BYTE* const litEnd = litPtr + dctx->litSize;
- const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
- const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
- const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
- DEBUGLOG(5, "ZSTD_decompressSequences_body");
- (void)frame;
-
- /* Regen sequences */
- if (nbSeq) {
- seqState_t seqState;
- dctx->fseEntropy = 1;
- { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
- RETURN_ERROR_IF(
- ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)),
- corruption_detected, "");
- ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
- ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
- ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
- assert(dst != NULL);
-
- ZSTD_STATIC_ASSERT(
- BIT_DStream_unfinished < BIT_DStream_completed &&
- BIT_DStream_endOfBuffer < BIT_DStream_completed &&
- BIT_DStream_completed < BIT_DStream_overflow);
-
-#if defined(__GNUC__) && defined(__x86_64__)
- __asm__(".p2align 6");
- __asm__("nop");
-# if __GNUC__ >= 7
- __asm__(".p2align 5");
- __asm__("nop");
- __asm__(".p2align 3");
-# else
- __asm__(".p2align 4");
- __asm__("nop");
- __asm__(".p2align 3");
-# endif
-#endif
-
- for ( ; ; ) {
- seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
- size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd);
-#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
- assert(!ZSTD_isError(oneSeqSize));
- if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
-#endif
- if (UNLIKELY(ZSTD_isError(oneSeqSize)))
- return oneSeqSize;
- DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
- op += oneSeqSize;
- if (UNLIKELY(!--nbSeq))
- break;
- BIT_reloadDStream(&(seqState.DStream));
- }
-
- /* check if reached exact end */
- DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq);
- RETURN_ERROR_IF(nbSeq, corruption_detected, "");
- RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, "");
- /* save reps for next block */
- { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
- }
-
- /* last literal segment */
- { size_t const lastLLSize = litEnd - litPtr;
- RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
- if (op != NULL) {
- ZSTD_memcpy(op, litPtr, lastLLSize);
- op += lastLLSize;
- }
- }
-
- return op-ostart;
-}
-
-static size_t
-ZSTD_decompressSequences_default(ZSTD_DCtx* dctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
-}
-
-static size_t
-ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
-}
-#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
-
-#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
-
-FORCE_INLINE_TEMPLATE size_t
-ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence,
- const BYTE* const prefixStart, const BYTE* const dictEnd)
-{
- prefetchPos += sequence.litLength;
- { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart;
- const BYTE* const match = matchBase + prefetchPos - sequence.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted.
- * No consequence though : memory address is only used for prefetching, not for dereferencing */
- PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
- }
- return prefetchPos + sequence.matchLength;
-}
-
-/* This decoding function employs prefetching
- * to reduce latency impact of cache misses.
- * It's generally employed when block contains a significant portion of long-distance matches
- * or when coupled with a "cold" dictionary */
-FORCE_INLINE_TEMPLATE size_t
-ZSTD_decompressSequencesLong_body(
- ZSTD_DCtx* dctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- const BYTE* ip = (const BYTE*)seqStart;
- const BYTE* const iend = ip + seqSize;
- BYTE* const ostart = (BYTE*)dst;
- BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ostart + maxDstSize;
- BYTE* op = ostart;
- const BYTE* litPtr = dctx->litPtr;
- const BYTE* litBufferEnd = dctx->litBufferEnd;
- const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
- const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart);
- const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
- (void)frame;
-
- /* Regen sequences */
- if (nbSeq) {
-#define STORED_SEQS 8
-#define STORED_SEQS_MASK (STORED_SEQS-1)
-#define ADVANCED_SEQS STORED_SEQS
- seq_t sequences[STORED_SEQS];
- int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
- seqState_t seqState;
- int seqNb;
- size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */
-
- dctx->fseEntropy = 1;
- { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
- assert(dst != NULL);
- assert(iend >= ip);
- RETURN_ERROR_IF(
- ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
- corruption_detected, "");
- ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
- ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
- ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
-
- /* prepare in advance */
- for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNb<seqAdvance); seqNb++) {
- seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
- prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
- sequences[seqNb] = sequence;
- }
- RETURN_ERROR_IF(seqNb<seqAdvance, corruption_detected, "");
-
- /* decompress without stomping litBuffer */
- for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb < nbSeq); seqNb++) {
- seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
- size_t oneSeqSize;
-
- if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd)
- {
- /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */
- const size_t leftoverLit = dctx->litBufferEnd - litPtr;
- if (leftoverLit)
- {
- RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
- ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
- sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit;
- op += leftoverLit;
- }
- litPtr = dctx->litExtraBuffer;
- litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
- dctx->litBufferLocation = ZSTD_not_in_dst;
- oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
-#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
- assert(!ZSTD_isError(oneSeqSize));
- if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
-#endif
- if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
-
- prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
- sequences[seqNb & STORED_SEQS_MASK] = sequence;
- op += oneSeqSize;
- }
- else
- {
- /* lit buffer is either wholly contained in first or second split, or not split at all*/
- oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
- ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
- ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
-#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
- assert(!ZSTD_isError(oneSeqSize));
- if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
-#endif
- if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
-
- prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
- sequences[seqNb & STORED_SEQS_MASK] = sequence;
- op += oneSeqSize;
- }
- }
- RETURN_ERROR_IF(seqNb<nbSeq, corruption_detected, "");
-
- /* finish queue */
- seqNb -= seqAdvance;
- for ( ; seqNb<nbSeq ; seqNb++) {
- seq_t *sequence = &(sequences[seqNb&STORED_SEQS_MASK]);
- if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd)
- {
- const size_t leftoverLit = dctx->litBufferEnd - litPtr;
- if (leftoverLit)
- {
- RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
- ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
- sequence->litLength -= leftoverLit;
- op += leftoverLit;
- }
- litPtr = dctx->litExtraBuffer;
- litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
- dctx->litBufferLocation = ZSTD_not_in_dst;
- {
- size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
-#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
- assert(!ZSTD_isError(oneSeqSize));
- if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
-#endif
- if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
- op += oneSeqSize;
- }
- }
- else
- {
- size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
- ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
- ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
-#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
- assert(!ZSTD_isError(oneSeqSize));
- if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
-#endif
- if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
- op += oneSeqSize;
- }
- }
-
- /* save reps for next block */
- { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
- }
-
- /* last literal segment */
- if (dctx->litBufferLocation == ZSTD_split) /* first deplete literal buffer in dst, then copy litExtraBuffer */
- {
- size_t const lastLLSize = litBufferEnd - litPtr;
- RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
- if (op != NULL) {
- ZSTD_memmove(op, litPtr, lastLLSize);
- op += lastLLSize;
- }
- litPtr = dctx->litExtraBuffer;
- litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
- }
- { size_t const lastLLSize = litBufferEnd - litPtr;
- RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
- if (op != NULL) {
- ZSTD_memmove(op, litPtr, lastLLSize);
- op += lastLLSize;
- }
- }
-
- return op-ostart;
-}
-
-static size_t
-ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
-}
-#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
-
-
-
-#if DYNAMIC_BMI2
-
-#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
-static BMI2_TARGET_ATTRIBUTE size_t
-DONT_VECTORIZE
-ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
-}
-static BMI2_TARGET_ATTRIBUTE size_t
-DONT_VECTORIZE
-ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
-}
-#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
-
-#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
-static BMI2_TARGET_ATTRIBUTE size_t
-ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
-}
-#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
-
-#endif /* DYNAMIC_BMI2 */
-
-typedef size_t (*ZSTD_decompressSequences_t)(
- ZSTD_DCtx* dctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame);
-
-#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
-static size_t
-ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- DEBUGLOG(5, "ZSTD_decompressSequences");
-#if DYNAMIC_BMI2
- if (ZSTD_DCtx_get_bmi2(dctx)) {
- return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
- }
-#endif
- return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
-}
-static size_t
-ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer");
-#if DYNAMIC_BMI2
- if (ZSTD_DCtx_get_bmi2(dctx)) {
- return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
- }
-#endif
- return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
-}
-#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
-
-
-#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
-/* ZSTD_decompressSequencesLong() :
- * decompression function triggered when a minimum share of offsets is considered "long",
- * aka out of cache.
- * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance".
- * This function will try to mitigate main memory latency through the use of prefetching */
-static size_t
-ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
- void* dst, size_t maxDstSize,
- const void* seqStart, size_t seqSize, int nbSeq,
- const ZSTD_longOffset_e isLongOffset,
- const int frame)
-{
- DEBUGLOG(5, "ZSTD_decompressSequencesLong");
-#if DYNAMIC_BMI2
- if (ZSTD_DCtx_get_bmi2(dctx)) {
- return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
- }
-#endif
- return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
-}
-#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
-
-
-
-#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
- !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
-/* ZSTD_getLongOffsetsShare() :
- * condition : offTable must be valid
- * @return : "share" of long offsets (arbitrarily defined as > (1<<23))
- * compared to maximum possible of (1<<OffFSELog) */
-static unsigned
-ZSTD_getLongOffsetsShare(const ZSTD_seqSymbol* offTable)
-{
- const void* ptr = offTable;
- U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
- const ZSTD_seqSymbol* table = offTable + 1;
- U32 const max = 1 << tableLog;
- U32 u, total = 0;
- DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
-
- assert(max <= (1 << OffFSELog)); /* max not too large */
- for (u=0; u<max; u++) {
- if (table[u].nbAdditionalBits > 22) total += 1;
- }
-
- assert(tableLog <= OffFSELog);
- total <<= (OffFSELog - tableLog); /* scale to OffFSELog */
-
- return total;
-}
-#endif
-
-size_t
-ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize, const int frame, const streaming_operation streaming)
-{ /* blockType == blockCompressed */
- const BYTE* ip = (const BYTE*)src;
- /* isLongOffset must be true if there are long offsets.
- * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN.
- * We don't expect that to be the case in 64-bit mode.
- * In block mode, window size is not known, so we have to be conservative.
- * (note: but it could be evaluated from current-lowLimit)
- */
- ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))));
- DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
-
- RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
-
- /* Decode literals section */
- { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
- DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize);
- if (ZSTD_isError(litCSize)) return litCSize;
- ip += litCSize;
- srcSize -= litCSize;
- }
-
- /* Build Decoding Tables */
- {
- /* These macros control at build-time which decompressor implementation
- * we use. If neither is defined, we do some inspection and dispatch at
- * runtime.
- */
-#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
- !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
- int usePrefetchDecoder = dctx->ddictIsCold;
-#endif
- int nbSeq;
- size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize);
- if (ZSTD_isError(seqHSize)) return seqHSize;
- ip += seqHSize;
- srcSize -= seqHSize;
-
- RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
-
-#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
- !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
- if ( !usePrefetchDecoder
- && (!frame || (dctx->fParams.windowSize > (1<<24)))
- && (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */
- U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr);
- U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
- usePrefetchDecoder = (shareLongOffsets >= minShare);
- }
-#endif
-
- dctx->ddictIsCold = 0;
-
-#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
- !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
- if (usePrefetchDecoder)
-#endif
-#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
- return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
-#endif
-
-#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
- /* else */
- if (dctx->litBufferLocation == ZSTD_split)
- return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
- else
- return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
-#endif
- }
-}
-
-
-void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize)
-{
- if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */
- dctx->dictEnd = dctx->previousDstEnd;
- dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
- dctx->prefixStart = dst;
- dctx->previousDstEnd = dst;
- }
-}
-
-
-size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize)
-{
- size_t dSize;
- ZSTD_checkContinuity(dctx, dst, dstCapacity);
- dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0, not_streaming);
- dctx->previousDstEnd = (char*)dst + dSize;
- return dSize;
-}
stage1/zstd/lib/decompress/zstd_decompress_block.h
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-
-#ifndef ZSTD_DEC_BLOCK_H
-#define ZSTD_DEC_BLOCK_H
-
-/*-*******************************************************
- * Dependencies
- *********************************************************/
-#include "../common/zstd_deps.h" /* size_t */
-#include "../zstd.h" /* DCtx, and some public functions */
-#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */
-#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
-
-
-/* === Prototypes === */
-
-/* note: prototypes already published within `zstd.h` :
- * ZSTD_decompressBlock()
- */
-
-/* note: prototypes already published within `zstd_internal.h` :
- * ZSTD_getcBlockSize()
- * ZSTD_decodeSeqHeaders()
- */
-
-
- /* Streaming state is used to inform allocation of the literal buffer */
-typedef enum {
- not_streaming = 0,
- is_streaming = 1
-} streaming_operation;
-
-/* ZSTD_decompressBlock_internal() :
- * decompress block, starting at `src`,
- * into destination buffer `dst`.
- * @return : decompressed block size,
- * or an error code (which can be tested using ZSTD_isError())
- */
-size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize, const int frame, const streaming_operation streaming);
-
-/* ZSTD_buildFSETable() :
- * generate FSE decoding table for one symbol (ll, ml or off)
- * this function must be called with valid parameters only
- * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.)
- * in which case it cannot fail.
- * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is
- * defined in zstd_decompress_internal.h.
- * Internal use only.
- */
-void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
- const short* normalizedCounter, unsigned maxSymbolValue,
- const U32* baseValue, const U8* nbAdditionalBits,
- unsigned tableLog, void* wksp, size_t wkspSize,
- int bmi2);
-
-
-#endif /* ZSTD_DEC_BLOCK_H */
stage1/zstd/lib/decompress/zstd_decompress_internal.h
@@ -1,236 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-
-/* zstd_decompress_internal:
- * objects and definitions shared within lib/decompress modules */
-
- #ifndef ZSTD_DECOMPRESS_INTERNAL_H
- #define ZSTD_DECOMPRESS_INTERNAL_H
-
-
-/*-*******************************************************
- * Dependencies
- *********************************************************/
-#include "../common/mem.h" /* BYTE, U16, U32 */
-#include "../common/zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */
-
-
-
-/*-*******************************************************
- * Constants
- *********************************************************/
-static UNUSED_ATTR const U32 LL_base[MaxLL+1] = {
- 0, 1, 2, 3, 4, 5, 6, 7,
- 8, 9, 10, 11, 12, 13, 14, 15,
- 16, 18, 20, 22, 24, 28, 32, 40,
- 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
- 0x2000, 0x4000, 0x8000, 0x10000 };
-
-static UNUSED_ATTR const U32 OF_base[MaxOff+1] = {
- 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
- 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
- 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
- 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };
-
-static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = {
- 0, 1, 2, 3, 4, 5, 6, 7,
- 8, 9, 10, 11, 12, 13, 14, 15,
- 16, 17, 18, 19, 20, 21, 22, 23,
- 24, 25, 26, 27, 28, 29, 30, 31 };
-
-static UNUSED_ATTR const U32 ML_base[MaxML+1] = {
- 3, 4, 5, 6, 7, 8, 9, 10,
- 11, 12, 13, 14, 15, 16, 17, 18,
- 19, 20, 21, 22, 23, 24, 25, 26,
- 27, 28, 29, 30, 31, 32, 33, 34,
- 35, 37, 39, 41, 43, 47, 51, 59,
- 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
- 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
-
-
-/*-*******************************************************
- * Decompression types
- *********************************************************/
- typedef struct {
- U32 fastMode;
- U32 tableLog;
- } ZSTD_seqSymbol_header;
-
- typedef struct {
- U16 nextState;
- BYTE nbAdditionalBits;
- BYTE nbBits;
- U32 baseValue;
- } ZSTD_seqSymbol;
-
- #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
-
-#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64))
-#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32))
-
-typedef struct {
- ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
- ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
- ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
- HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
- U32 rep[ZSTD_REP_NUM];
- U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
-} ZSTD_entropyDTables_t;
-
-typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
- ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
- ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
- ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
-
-typedef enum { zdss_init=0, zdss_loadHeader,
- zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
-
-typedef enum {
- ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */
- ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */
- ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */
-} ZSTD_dictUses_e;
-
-/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */
-typedef struct {
- const ZSTD_DDict** ddictPtrTable;
- size_t ddictPtrTableSize;
- size_t ddictPtrCount;
-} ZSTD_DDictHashSet;
-
-#ifndef ZSTD_DECODER_INTERNAL_BUFFER
-# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16)
-#endif
-
-#define ZSTD_LBMIN 64
-#define ZSTD_LBMAX (128 << 10)
-
-/* extra buffer, compensates when dst is not large enough to store litBuffer */
-#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX)
-
-typedef enum {
- ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */
- ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */
- ZSTD_split = 2 /* Split between litExtraBuffer and dst */
-} ZSTD_litLocation_e;
-
-struct ZSTD_DCtx_s
-{
- const ZSTD_seqSymbol* LLTptr;
- const ZSTD_seqSymbol* MLTptr;
- const ZSTD_seqSymbol* OFTptr;
- const HUF_DTable* HUFptr;
- ZSTD_entropyDTables_t entropy;
- U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */
- const void* previousDstEnd; /* detect continuity */
- const void* prefixStart; /* start of current segment */
- const void* virtualStart; /* virtual start of previous segment if it was just before current one */
- const void* dictEnd; /* end of previous segment */
- size_t expected;
- ZSTD_frameHeader fParams;
- U64 processedCSize;
- U64 decodedSize;
- blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
- ZSTD_dStage stage;
- U32 litEntropy;
- U32 fseEntropy;
- XXH64_state_t xxhState;
- size_t headerSize;
- ZSTD_format_e format;
- ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */
- U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */
- const BYTE* litPtr;
- ZSTD_customMem customMem;
- size_t litSize;
- size_t rleSize;
- size_t staticSize;
-#if DYNAMIC_BMI2 != 0
- int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
-#endif
-
- /* dictionary */
- ZSTD_DDict* ddictLocal;
- const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
- U32 dictID;
- int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
- ZSTD_dictUses_e dictUses;
- ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */
- ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
-
- /* streaming */
- ZSTD_dStreamStage streamStage;
- char* inBuff;
- size_t inBuffSize;
- size_t inPos;
- size_t maxWindowSize;
- char* outBuff;
- size_t outBuffSize;
- size_t outStart;
- size_t outEnd;
- size_t lhSize;
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
- void* legacyContext;
- U32 previousLegacyVersion;
- U32 legacyVersion;
-#endif
- U32 hostageByte;
- int noForwardProgress;
- ZSTD_bufferMode_e outBufferMode;
- ZSTD_outBuffer expectedOutBuffer;
-
- /* workspace */
- BYTE* litBuffer;
- const BYTE* litBufferEnd;
- ZSTD_litLocation_e litBufferLocation;
- BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */
- BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
-
- size_t oversizedDuration;
-
-#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
- void const* dictContentBeginForFuzzing;
- void const* dictContentEndForFuzzing;
-#endif
-
- /* Tracing */
-#if ZSTD_TRACE
- ZSTD_TraceCtx traceCtx;
-#endif
-}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
-
-MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) {
-#if DYNAMIC_BMI2 != 0
- return dctx->bmi2;
-#else
- (void)dctx;
- return 0;
-#endif
-}
-
-/*-*******************************************************
- * Shared internal functions
- *********************************************************/
-
-/*! ZSTD_loadDEntropy() :
- * dict : must point at beginning of a valid zstd dictionary.
- * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */
-size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
- const void* const dict, size_t const dictSize);
-
-/*! ZSTD_checkContinuity() :
- * check if next `dst` follows previous position, where decompression ended.
- * If yes, do nothing (continue on current segment).
- * If not, classify previous segment as "external dictionary", and start a new segment.
- * This function cannot fail. */
-void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize);
-
-
-#endif /* ZSTD_DECOMPRESS_INTERNAL_H */
stage1/zstd/lib/zstd.h
@@ -1,2575 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-#ifndef ZSTD_H_235446
-#define ZSTD_H_235446
-
-/* ====== Dependency ======*/
-#include <limits.h> /* INT_MAX */
-#include <stddef.h> /* size_t */
-
-
-/* ===== ZSTDLIB_API : control library symbols visibility ===== */
-#ifndef ZSTDLIB_VISIBLE
-# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
-# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
-# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
-# else
-# define ZSTDLIB_VISIBLE
-# define ZSTDLIB_HIDDEN
-# endif
-#endif
-#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
-#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
-#else
-# define ZSTDLIB_API ZSTDLIB_VISIBLE
-#endif
-
-
-/*******************************************************************************
- Introduction
-
- zstd, short for Zstandard, is a fast lossless compression algorithm, targeting
- real-time compression scenarios at zlib-level and better compression ratios.
- The zstd compression library provides in-memory compression and decompression
- functions.
-
- The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
- which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
- caution, as they require more memory. The library also offers negative
- compression levels, which extend the range of speed vs. ratio preferences.
- The lower the level, the faster the speed (at the cost of compression).
-
- Compression can be done in:
- - a single step (described as Simple API)
- - a single step, reusing a context (described as Explicit context)
- - unbounded multiple steps (described as Streaming compression)
-
- The compression ratio achievable on small data can be highly improved using
- a dictionary. Dictionary compression can be performed in:
- - a single step (described as Simple dictionary API)
- - a single step, reusing a dictionary (described as Bulk-processing
- dictionary API)
-
- Advanced experimental functions can be accessed using
- `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.
-
- Advanced experimental APIs should never be used with a dynamically-linked
- library. They are not "stable"; their definitions or signatures may change in
- the future. Only static linking is allowed.
-*******************************************************************************/
-
-/*------ Version ------*/
-#define ZSTD_VERSION_MAJOR 1
-#define ZSTD_VERSION_MINOR 5
-#define ZSTD_VERSION_RELEASE 2
-#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
-
-/*! ZSTD_versionNumber() :
- * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */
-ZSTDLIB_API unsigned ZSTD_versionNumber(void);
-
-#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
-#define ZSTD_QUOTE(str) #str
-#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
-#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
-
-/*! ZSTD_versionString() :
- * Return runtime library version, like "1.4.5". Requires v1.3.0+. */
-ZSTDLIB_API const char* ZSTD_versionString(void);
-
-/* *************************************
- * Default constant
- ***************************************/
-#ifndef ZSTD_CLEVEL_DEFAULT
-# define ZSTD_CLEVEL_DEFAULT 3
-#endif
-
-/* *************************************
- * Constants
- ***************************************/
-
-/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
-#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
-#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */
-#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
-#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0
-
-#define ZSTD_BLOCKSIZELOG_MAX 17
-#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX)
-
-
-/***************************************
-* Simple API
-***************************************/
-/*! ZSTD_compress() :
- * Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
- * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
- * @return : compressed size written into `dst` (<= `dstCapacity),
- * or an error code if it fails (which can be tested using ZSTD_isError()). */
-ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- int compressionLevel);
-
-/*! ZSTD_decompress() :
- * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
- * `dstCapacity` is an upper bound of originalSize to regenerate.
- * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
- * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
- * or an errorCode if it fails (which can be tested using ZSTD_isError()). */
-ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
- const void* src, size_t compressedSize);
-
-/*! ZSTD_getFrameContentSize() : requires v1.3.0+
- * `src` should point to the start of a ZSTD encoded frame.
- * `srcSize` must be at least as large as the frame header.
- * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
- * @return : - decompressed size of `src` frame content, if known
- * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
- * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
- * note 1 : a 0 return value means the frame is valid but "empty".
- * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
- * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
- * In which case, it's necessary to use streaming mode to decompress data.
- * Optionally, application can rely on some implicit limit,
- * as ZSTD_decompress() only needs an upper bound of decompressed size.
- * (For example, data could be necessarily cut into blocks <= 16 KB).
- * note 3 : decompressed size is always present when compression is completed using single-pass functions,
- * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
- * note 4 : decompressed size can be very large (64-bits value),
- * potentially larger than what local system can handle as a single memory segment.
- * In which case, it's necessary to use streaming mode to decompress data.
- * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
- * Always ensure return value fits within application's authorized limits.
- * Each application can set its own limits.
- * note 6 : This function replaces ZSTD_getDecompressedSize() */
-#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
-#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
-ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
-
-/*! ZSTD_getDecompressedSize() :
- * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
- * Both functions work the same way, but ZSTD_getDecompressedSize() blends
- * "empty", "unknown" and "error" results to the same return value (0),
- * while ZSTD_getFrameContentSize() gives them separate return values.
- * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
-ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
-
-/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
- * `src` should point to the start of a ZSTD frame or skippable frame.
- * `srcSize` must be >= first frame size
- * @return : the compressed size of the first frame starting at `src`,
- * suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
- * or an error code if input is invalid */
-ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
-
-
-/*====== Helper functions ======*/
-#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
-ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
-ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
-ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */
-ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
-ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */
-
-
-/***************************************
-* Explicit context
-***************************************/
-/*= Compression context
- * When compressing many times,
- * it is recommended to allocate a context just once,
- * and re-use it for each successive compression operation.
- * This will make workload friendlier for system's memory.
- * Note : re-using context is just a speed / resource optimization.
- * It doesn't change the compression ratio, which remains identical.
- * Note 2 : In multi-threaded environments,
- * use one different context per thread for parallel execution.
- */
-typedef struct ZSTD_CCtx_s ZSTD_CCtx;
-ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
-ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */
-
-/*! ZSTD_compressCCtx() :
- * Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
- * Important : in order to behave similarly to `ZSTD_compress()`,
- * this function compresses at requested compression level,
- * __ignoring any other parameter__ .
- * If any advanced parameter was set using the advanced API,
- * they will all be reset. Only `compressionLevel` remains.
- */
-ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- int compressionLevel);
-
-/*= Decompression context
- * When decompressing many times,
- * it is recommended to allocate a context only once,
- * and re-use it for each successive compression operation.
- * This will make workload friendlier for system's memory.
- * Use one context per thread for parallel execution. */
-typedef struct ZSTD_DCtx_s ZSTD_DCtx;
-ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
-ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */
-
-/*! ZSTD_decompressDCtx() :
- * Same as ZSTD_decompress(),
- * requires an allocated ZSTD_DCtx.
- * Compatible with sticky parameters.
- */
-ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize);
-
-
-/*********************************************
-* Advanced compression API (Requires v1.4.0+)
-**********************************************/
-
-/* API design :
- * Parameters are pushed one by one into an existing context,
- * using ZSTD_CCtx_set*() functions.
- * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
- * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
- * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .
- *
- * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
- *
- * This API supersedes all other "advanced" API entry points in the experimental section.
- * In the future, we expect to remove from experimental API entry points which are redundant with this API.
- */
-
-
-/* Compression strategies, listed from fastest to strongest */
-typedef enum { ZSTD_fast=1,
- ZSTD_dfast=2,
- ZSTD_greedy=3,
- ZSTD_lazy=4,
- ZSTD_lazy2=5,
- ZSTD_btlazy2=6,
- ZSTD_btopt=7,
- ZSTD_btultra=8,
- ZSTD_btultra2=9
- /* note : new strategies _might_ be added in the future.
- Only the order (from fast to strong) is guaranteed */
-} ZSTD_strategy;
-
-typedef enum {
-
- /* compression parameters
- * Note: When compressing with a ZSTD_CDict these parameters are superseded
- * by the parameters used to construct the ZSTD_CDict.
- * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
- ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
- * Note that exact compression parameters are dynamically determined,
- * depending on both compression level and srcSize (when known).
- * Default level is ZSTD_CLEVEL_DEFAULT==3.
- * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
- * Note 1 : it's possible to pass a negative compression level.
- * Note 2 : setting a level does not automatically set all other compression parameters
- * to default. Setting this will however eventually dynamically impact the compression
- * parameters which have not been manually set. The manually set
- * ones will 'stick'. */
- /* Advanced compression parameters :
- * It's possible to pin down compression parameters to some specific values.
- * In which case, these values are no longer dynamically selected by the compressor */
- ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
- * This will set a memory budget for streaming decompression,
- * with larger values requiring more memory
- * and typically compressing more.
- * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
- * Special: value 0 means "use default windowLog".
- * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
- * requires explicitly allowing such size at streaming decompression stage. */
- ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
- * Resulting memory usage is (1 << (hashLog+2)).
- * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
- * Larger tables improve compression ratio of strategies <= dFast,
- * and improve speed of strategies > dFast.
- * Special: value 0 means "use default hashLog". */
- ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2.
- * Resulting memory usage is (1 << (chainLog+2)).
- * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
- * Larger tables result in better and slower compression.
- * This parameter is useless for "fast" strategy.
- * It's still useful when using "dfast" strategy,
- * in which case it defines a secondary probe table.
- * Special: value 0 means "use default chainLog". */
- ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
- * More attempts result in better and slower compression.
- * This parameter is useless for "fast" and "dFast" strategies.
- * Special: value 0 means "use default searchLog". */
- ZSTD_c_minMatch=105, /* Minimum size of searched matches.
- * Note that Zstandard can still find matches of smaller size,
- * it just tweaks its search algorithm to look for this size and larger.
- * Larger values increase compression and decompression speed, but decrease ratio.
- * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
- * Note that currently, for all strategies < btopt, effective minimum is 4.
- * , for all strategies > fast, effective maximum is 6.
- * Special: value 0 means "use default minMatchLength". */
- ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
- * For strategies btopt, btultra & btultra2:
- * Length of Match considered "good enough" to stop search.
- * Larger values make compression stronger, and slower.
- * For strategy fast:
- * Distance between match sampling.
- * Larger values make compression faster, and weaker.
- * Special: value 0 means "use default targetLength". */
- ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition.
- * The higher the value of selected strategy, the more complex it is,
- * resulting in stronger and slower compression.
- * Special: value 0 means "use default strategy". */
- /* LDM mode parameters */
- ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
- * This parameter is designed to improve compression ratio
- * for large inputs, by finding large matches at long distance.
- * It increases memory usage and window size.
- * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
- * except when expressly set to a different value.
- * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
- * compression strategy >= ZSTD_btopt (== compression level 16+) */
- ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
- * Larger values increase memory usage and compression ratio,
- * but decrease compression speed.
- * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
- * default: windowlog - 7.
- * Special: value 0 means "automatically determine hashlog". */
- ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher.
- * Larger/too small values usually decrease compression ratio.
- * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
- * Special: value 0 means "use default value" (default: 64). */
- ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
- * Larger values improve collision resolution but decrease compression speed.
- * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
- * Special: value 0 means "use default value" (default: 3). */
- ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
- * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
- * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
- * Larger values improve compression speed.
- * Deviating far from default value will likely result in a compression ratio decrease.
- * Special: value 0 means "automatically determine hashRateLog". */
-
- /* frame parameters */
- ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
- * Content size must be known at the beginning of compression.
- * This is automatically the case when using ZSTD_compress2(),
- * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
- ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
- ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
-
- /* multi-threading parameters */
- /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
- * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
- * In a situation where it's unknown if the linked library supports multi-threading or not,
- * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
- */
- ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
- * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
- * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
- * while compression is performed in parallel, within worker thread(s).
- * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
- * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
- * More workers improve speed, but also increase memory usage.
- * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
- * compression is performed inside Caller's thread, and all invocations are blocking */
- ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
- * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
- * 0 means default, which is dynamically determined based on compression parameters.
- * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
- * The minimum size is automatically and transparently enforced. */
- ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
- * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
- * It helps preserve compression ratio, while each job is compressed in parallel.
- * This value is enforced only when nbWorkers >= 1.
- * Larger values increase compression ratio, but decrease speed.
- * Possible values range from 0 to 9 :
- * - 0 means "default" : value will be determined by the library, depending on strategy
- * - 1 means "no overlap"
- * - 9 means "full overlap", using a full window size.
- * Each intermediate rank increases/decreases load size by a factor 2 :
- * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default
- * default value varies between 6 and 9, depending on strategy */
-
- /* note : additional experimental parameters are also available
- * within the experimental section of the API.
- * At the time of this writing, they include :
- * ZSTD_c_rsyncable
- * ZSTD_c_format
- * ZSTD_c_forceMaxWindow
- * ZSTD_c_forceAttachDict
- * ZSTD_c_literalCompressionMode
- * ZSTD_c_targetCBlockSize
- * ZSTD_c_srcSizeHint
- * ZSTD_c_enableDedicatedDictSearch
- * ZSTD_c_stableInBuffer
- * ZSTD_c_stableOutBuffer
- * ZSTD_c_blockDelimiters
- * ZSTD_c_validateSequences
- * ZSTD_c_useBlockSplitter
- * ZSTD_c_useRowMatchFinder
- * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
- * note : never ever use experimentalParam? names directly;
- * also, the enums values themselves are unstable and can still change.
- */
- ZSTD_c_experimentalParam1=500,
- ZSTD_c_experimentalParam2=10,
- ZSTD_c_experimentalParam3=1000,
- ZSTD_c_experimentalParam4=1001,
- ZSTD_c_experimentalParam5=1002,
- ZSTD_c_experimentalParam6=1003,
- ZSTD_c_experimentalParam7=1004,
- ZSTD_c_experimentalParam8=1005,
- ZSTD_c_experimentalParam9=1006,
- ZSTD_c_experimentalParam10=1007,
- ZSTD_c_experimentalParam11=1008,
- ZSTD_c_experimentalParam12=1009,
- ZSTD_c_experimentalParam13=1010,
- ZSTD_c_experimentalParam14=1011,
- ZSTD_c_experimentalParam15=1012
-} ZSTD_cParameter;
-
-typedef struct {
- size_t error;
- int lowerBound;
- int upperBound;
-} ZSTD_bounds;
-
-/*! ZSTD_cParam_getBounds() :
- * All parameters must belong to an interval with lower and upper bounds,
- * otherwise they will either trigger an error or be automatically clamped.
- * @return : a structure, ZSTD_bounds, which contains
- * - an error status field, which must be tested using ZSTD_isError()
- * - lower and upper bounds, both inclusive
- */
-ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
-
-/*! ZSTD_CCtx_setParameter() :
- * Set one compression parameter, selected by enum ZSTD_cParameter.
- * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
- * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
- * Setting a parameter is generally only possible during frame initialization (before starting compression).
- * Exception : when using multi-threading mode (nbWorkers >= 1),
- * the following parameters can be updated _during_ compression (within same frame):
- * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
- * new parameters will be active for next job only (after a flush()).
- * @return : an error code (which can be tested using ZSTD_isError()).
- */
-ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
-
-/*! ZSTD_CCtx_setPledgedSrcSize() :
- * Total input data size to be compressed as a single frame.
- * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
- * This value will also be controlled at end of frame, and trigger an error if not respected.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
- * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
- * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
- * Note 2 : pledgedSrcSize is only valid once, for the next frame.
- * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
- * Note 3 : Whenever all input data is provided and consumed in a single round,
- * for example with ZSTD_compress2(),
- * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
- * this value is automatically overridden by srcSize instead.
- */
-ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
-
-typedef enum {
- ZSTD_reset_session_only = 1,
- ZSTD_reset_parameters = 2,
- ZSTD_reset_session_and_parameters = 3
-} ZSTD_ResetDirective;
-
-/*! ZSTD_CCtx_reset() :
- * There are 2 different things that can be reset, independently or jointly :
- * - The session : will stop compressing current frame, and make CCtx ready to start a new one.
- * Useful after an error, or to interrupt any ongoing compression.
- * Any internal data not yet flushed is cancelled.
- * Compression parameters and dictionary remain unchanged.
- * They will be used to compress next frame.
- * Resetting session never fails.
- * - The parameters : changes all parameters back to "default".
- * This removes any reference to any dictionary too.
- * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
- * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
- * - Both : similar to resetting the session, followed by resetting parameters.
- */
-ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
-
-/*! ZSTD_compress2() :
- * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
- * ZSTD_compress2() always starts a new frame.
- * Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
- * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
- * - The function is always blocking, returns when compression is completed.
- * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
- * @return : compressed size written into `dst` (<= `dstCapacity),
- * or an error code if it fails (which can be tested using ZSTD_isError()).
- */
-ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize);
-
-
-/***********************************************
-* Advanced decompression API (Requires v1.4.0+)
-************************************************/
-
-/* The advanced API pushes parameters one by one into an existing DCtx context.
- * Parameters are sticky, and remain valid for all following frames
- * using the same DCtx context.
- * It's possible to reset parameters to default values using ZSTD_DCtx_reset().
- * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream().
- * Therefore, no new decompression function is necessary.
- */
-
-typedef enum {
-
- ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
- * the streaming API will refuse to allocate memory buffer
- * in order to protect the host from unreasonable memory requirements.
- * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
- * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
- * Special: value 0 means "use default maximum windowLog". */
-
- /* note : additional experimental parameters are also available
- * within the experimental section of the API.
- * At the time of this writing, they include :
- * ZSTD_d_format
- * ZSTD_d_stableOutBuffer
- * ZSTD_d_forceIgnoreChecksum
- * ZSTD_d_refMultipleDDicts
- * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
- * note : never ever use experimentalParam? names directly
- */
- ZSTD_d_experimentalParam1=1000,
- ZSTD_d_experimentalParam2=1001,
- ZSTD_d_experimentalParam3=1002,
- ZSTD_d_experimentalParam4=1003
-
-} ZSTD_dParameter;
-
-/*! ZSTD_dParam_getBounds() :
- * All parameters must belong to an interval with lower and upper bounds,
- * otherwise they will either trigger an error or be automatically clamped.
- * @return : a structure, ZSTD_bounds, which contains
- * - an error status field, which must be tested using ZSTD_isError()
- * - both lower and upper bounds, inclusive
- */
-ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
-
-/*! ZSTD_DCtx_setParameter() :
- * Set one compression parameter, selected by enum ZSTD_dParameter.
- * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
- * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
- * Setting a parameter is only possible during frame initialization (before starting decompression).
- * @return : 0, or an error code (which can be tested using ZSTD_isError()).
- */
-ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
-
-/*! ZSTD_DCtx_reset() :
- * Return a DCtx to clean state.
- * Session and parameters can be reset jointly or separately.
- * Parameters can only be reset when no active frame is being decompressed.
- * @return : 0, or an error code, which can be tested with ZSTD_isError()
- */
-ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
-
-
-/****************************
-* Streaming
-****************************/
-
-typedef struct ZSTD_inBuffer_s {
- const void* src; /**< start of input buffer */
- size_t size; /**< size of input buffer */
- size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
-} ZSTD_inBuffer;
-
-typedef struct ZSTD_outBuffer_s {
- void* dst; /**< start of output buffer */
- size_t size; /**< size of output buffer */
- size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
-} ZSTD_outBuffer;
-
-
-
-/*-***********************************************************************
-* Streaming compression - HowTo
-*
-* A ZSTD_CStream object is required to track streaming operation.
-* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
-* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
-* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
-*
-* For parallel execution, use one separate ZSTD_CStream per thread.
-*
-* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
-*
-* Parameters are sticky : when starting a new compression on the same context,
-* it will re-use the same sticky parameters as previous compression session.
-* When in doubt, it's recommended to fully initialize the context before usage.
-* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
-* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
-* set more specific parameters, the pledged source size, or load a dictionary.
-*
-* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to
-* consume input stream. The function will automatically update both `pos`
-* fields within `input` and `output`.
-* Note that the function may not consume the entire input, for example, because
-* the output buffer is already full, in which case `input.pos < input.size`.
-* The caller must check if input has been entirely consumed.
-* If not, the caller must make some room to receive more compressed data,
-* and then present again remaining input data.
-* note: ZSTD_e_continue is guaranteed to make some forward progress when called,
-* but doesn't guarantee maximal forward progress. This is especially relevant
-* when compressing with multiple threads. The call won't block if it can
-* consume some input, but if it can't it will wait for some, but not all,
-* output to be flushed.
-* @return : provides a minimum amount of data remaining to be flushed from internal buffers
-* or an error code, which can be tested using ZSTD_isError().
-*
-* At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
-* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
-* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).
-* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
-* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
-* operation.
-* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will
-* block until the flush is complete or the output buffer is full.
-* @return : 0 if internal buffers are entirely flushed,
-* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
-* or an error code, which can be tested using ZSTD_isError().
-*
-* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.
-* It will perform a flush and write frame epilogue.
-* The epilogue is required for decoders to consider a frame completed.
-* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
-* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
-* start a new frame.
-* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will
-* block until the flush is complete or the output buffer is full.
-* @return : 0 if frame fully completed and fully flushed,
-* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
-* or an error code, which can be tested using ZSTD_isError().
-*
-* *******************************************************************/
-
-typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
- /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
-/*===== ZSTD_CStream management functions =====*/
-ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
-ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */
-
-/*===== Streaming compression functions =====*/
-typedef enum {
- ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
- ZSTD_e_flush=1, /* flush any data provided so far,
- * it creates (at least) one new block, that can be decoded immediately on reception;
- * frame will continue: any future data can still reference previously compressed data, improving compression.
- * note : multithreaded compression will block to flush as much output as possible. */
- ZSTD_e_end=2 /* flush any remaining data _and_ close current frame.
- * note that frame is only closed after compressed data is fully flushed (return value == 0).
- * After that point, any additional data starts a new frame.
- * note : each frame is independent (does not reference any content from previous frame).
- : note : multithreaded compression will block to flush as much output as possible. */
-} ZSTD_EndDirective;
-
-/*! ZSTD_compressStream2() : Requires v1.4.0+
- * Behaves about the same as ZSTD_compressStream, with additional control on end directive.
- * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
- * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
- * - output->pos must be <= dstCapacity, input->pos must be <= srcSize
- * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
- * - endOp must be a valid directive
- * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
- * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
- * and then immediately returns, just indicating that there is some data remaining to be flushed.
- * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
- * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
- * - @return provides a minimum amount of data remaining to be flushed from internal buffers
- * or an error code, which can be tested using ZSTD_isError().
- * if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
- * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
- * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
- * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
- * only ZSTD_e_end or ZSTD_e_flush operations are allowed.
- * Before starting a new compression job, or changing compression parameters,
- * it is required to fully flush internal buffers.
- */
-ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
- ZSTD_outBuffer* output,
- ZSTD_inBuffer* input,
- ZSTD_EndDirective endOp);
-
-
-/* These buffer sizes are softly recommended.
- * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
- * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
- * reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
- *
- * However, note that these recommendations are from the perspective of a C caller program.
- * If the streaming interface is invoked from some other language,
- * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
- * a major performance rule is to reduce crossing such interface to an absolute minimum.
- * It's not rare that performance ends being spent more into the interface, rather than compression itself.
- * In which cases, prefer using large buffers, as large as practical,
- * for both input and output, to reduce the nb of roundtrips.
- */
-ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
-ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
-
-
-/* *****************************************************************************
- * This following is a legacy streaming API, available since v1.0+ .
- * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
- * It is redundant, but remains fully supported.
- * Streaming in combination with advanced parameters and dictionary compression
- * can only be used through the new API.
- ******************************************************************************/
-
-/*!
- * Equivalent to:
- *
- * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
- * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
- */
-ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
-/*!
- * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
- * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
- * the next read size (if non-zero and not an error). ZSTD_compressStream2()
- * returns the minimum nb of bytes left to flush (if non-zero and not an error).
- */
-ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
-ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
-ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-
-
-/*-***************************************************************************
-* Streaming decompression - HowTo
-*
-* A ZSTD_DStream object is required to track streaming operations.
-* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
-* ZSTD_DStream objects can be re-used multiple times.
-*
-* Use ZSTD_initDStream() to start a new decompression operation.
-* @return : recommended first input size
-* Alternatively, use advanced API to set specific properties.
-*
-* Use ZSTD_decompressStream() repetitively to consume your input.
-* The function will update both `pos` fields.
-* If `input.pos < input.size`, some input has not been consumed.
-* It's up to the caller to present again remaining data.
-* The function tries to flush all data decoded immediately, respecting output buffer size.
-* If `output.pos < output.size`, decoder has flushed everything it could.
-* But if `output.pos == output.size`, there might be some data left within internal buffers.,
-* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
-* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
-* @return : 0 when a frame is completely decoded and fully flushed,
-* or an error code, which can be tested using ZSTD_isError(),
-* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
-* the return value is a suggested next input size (just a hint for better latency)
-* that will never request more than the remaining frame size.
-* *******************************************************************************/
-
-typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
- /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
-/*===== ZSTD_DStream management functions =====*/
-ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
-ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */
-
-/*===== Streaming decompression functions =====*/
-
-/* This function is redundant with the advanced API and equivalent to:
- *
- * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
- * ZSTD_DCtx_refDDict(zds, NULL);
- */
-ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
-
-ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-
-ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
-ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
-
-
-/**************************
-* Simple dictionary API
-***************************/
-/*! ZSTD_compress_usingDict() :
- * Compression at an explicit compression level using a Dictionary.
- * A dictionary can be any arbitrary data segment (also called a prefix),
- * or a buffer with specified information (see zdict.h).
- * Note : This function loads the dictionary, resulting in significant startup delay.
- * It's intended for a dictionary used only once.
- * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
-ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize,
- int compressionLevel);
-
-/*! ZSTD_decompress_usingDict() :
- * Decompression using a known Dictionary.
- * Dictionary must be identical to the one used during compression.
- * Note : This function loads the dictionary, resulting in significant startup delay.
- * It's intended for a dictionary used only once.
- * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
-ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize);
-
-
-/***********************************
- * Bulk processing dictionary API
- **********************************/
-typedef struct ZSTD_CDict_s ZSTD_CDict;
-
-/*! ZSTD_createCDict() :
- * When compressing multiple messages or blocks using the same dictionary,
- * it's recommended to digest the dictionary only once, since it's a costly operation.
- * ZSTD_createCDict() will create a state from digesting a dictionary.
- * The resulting state can be used for future compression operations with very limited startup cost.
- * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
- * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
- * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
- * in which case the only thing that it transports is the @compressionLevel.
- * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
- * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
-ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
- int compressionLevel);
-
-/*! ZSTD_freeCDict() :
- * Function frees memory allocated by ZSTD_createCDict().
- * If a NULL pointer is passed, no operation is performed. */
-ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
-
-/*! ZSTD_compress_usingCDict() :
- * Compression using a digested Dictionary.
- * Recommended when same dictionary is used multiple times.
- * Note : compression level is _decided at dictionary creation time_,
- * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
-ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTD_CDict* cdict);
-
-
-typedef struct ZSTD_DDict_s ZSTD_DDict;
-
-/*! ZSTD_createDDict() :
- * Create a digested dictionary, ready to start decompression operation without startup delay.
- * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
-ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
-
-/*! ZSTD_freeDDict() :
- * Function frees memory allocated with ZSTD_createDDict()
- * If a NULL pointer is passed, no operation is performed. */
-ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
-
-/*! ZSTD_decompress_usingDDict() :
- * Decompression using a digested Dictionary.
- * Recommended when same dictionary is used multiple times. */
-ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTD_DDict* ddict);
-
-
-/********************************
- * Dictionary helper functions
- *******************************/
-
-/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+
- * Provides the dictID stored within dictionary.
- * if @return == 0, the dictionary is not conformant with Zstandard specification.
- * It can still be loaded, but as a content-only dictionary. */
-ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
-
-/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+
- * Provides the dictID of the dictionary loaded into `cdict`.
- * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
- * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
-ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
-
-/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+
- * Provides the dictID of the dictionary loaded into `ddict`.
- * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
- * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
-ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
-
-/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+
- * Provides the dictID required to decompressed the frame stored within `src`.
- * If @return == 0, the dictID could not be decoded.
- * This could for one of the following reasons :
- * - The frame does not require a dictionary to be decoded (most common case).
- * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
- * Note : this use case also happens when using a non-conformant dictionary.
- * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
- * - This is not a Zstandard frame.
- * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
-ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
-
-
-/*******************************************************************************
- * Advanced dictionary and prefix API (Requires v1.4.0+)
- *
- * This API allows dictionaries to be used with ZSTD_compress2(),
- * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
- * only reset with the context is reset with ZSTD_reset_parameters or
- * ZSTD_reset_session_and_parameters. Prefixes are single-use.
- ******************************************************************************/
-
-
-/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+
- * Create an internal CDict from `dict` buffer.
- * Decompression will have to use same dictionary.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
- * meaning "return to no-dictionary mode".
- * Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
- * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
- * Note 2 : Loading a dictionary involves building tables.
- * It's also a CPU consuming operation, with non-negligible impact on latency.
- * Tables are dependent on compression parameters, and for this reason,
- * compression parameters can no longer be changed after loading a dictionary.
- * Note 3 :`dict` content will be copied internally.
- * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
- * In such a case, dictionary buffer must outlive its users.
- * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
- * to precisely select how dictionary content must be interpreted. */
-ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
-
-/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
- * Reference a prepared dictionary, to be used for all next compressed frames.
- * Note that compression parameters are enforced from within CDict,
- * and supersede any compression parameter previously set within CCtx.
- * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
- * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
- * The dictionary will remain valid for future compressed frames using same CCtx.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Special : Referencing a NULL CDict means "return to no-dictionary mode".
- * Note 1 : Currently, only one dictionary can be managed.
- * Referencing a new dictionary effectively "discards" any previous one.
- * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
-ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
-
-/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+
- * Reference a prefix (single-usage dictionary) for next compressed frame.
- * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
- * Decompression will need same prefix to properly regenerate data.
- * Compressing with a prefix is similar in outcome as performing a diff and compressing it,
- * but performs much faster, especially during decompression (compression speed is tunable with compression level).
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
- * Note 1 : Prefix buffer is referenced. It **must** outlive compression.
- * Its content must remain unmodified during compression.
- * Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
- * ensure that the window size is large enough to contain the entire source.
- * See ZSTD_c_windowLog.
- * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
- * It's a CPU consuming operation, with non-negligible impact on latency.
- * If there is a need to use the same prefix multiple times, consider loadDictionary instead.
- * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
- * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
-ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
- const void* prefix, size_t prefixSize);
-
-/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
- * Create an internal DDict from dict buffer,
- * to be used to decompress next frames.
- * The dictionary remains valid for all future frames, until explicitly invalidated.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
- * meaning "return to no-dictionary mode".
- * Note 1 : Loading a dictionary involves building tables,
- * which has a non-negligible impact on CPU usage and latency.
- * It's recommended to "load once, use many times", to amortize the cost
- * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
- * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
- * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
- * how dictionary content is loaded and interpreted.
- */
-ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-
-/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+
- * Reference a prepared dictionary, to be used to decompress next frames.
- * The dictionary remains active for decompression of future frames using same DCtx.
- *
- * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
- * will store the DDict references in a table, and the DDict used for decompression
- * will be determined at decompression time, as per the dict ID in the frame.
- * The memory for the table is allocated on the first call to refDDict, and can be
- * freed with ZSTD_freeDCtx().
- *
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Note 1 : Currently, only one dictionary can be managed.
- * Referencing a new dictionary effectively "discards" any previous one.
- * Special: referencing a NULL DDict means "return to no-dictionary mode".
- * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
- */
-ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
-
-/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+
- * Reference a prefix (single-usage dictionary) to decompress next frame.
- * This is the reverse operation of ZSTD_CCtx_refPrefix(),
- * and must use the same prefix as the one used during compression.
- * Prefix is **only used once**. Reference is discarded at end of frame.
- * End of frame is reached when ZSTD_decompressStream() returns 0.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
- * Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
- * Prefix buffer must remain unmodified up to the end of frame,
- * reached when ZSTD_decompressStream() returns 0.
- * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
- * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
- * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
- * A full dictionary is more costly, as it requires building tables.
- */
-ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
- const void* prefix, size_t prefixSize);
-
-/* === Memory management === */
-
-/*! ZSTD_sizeof_*() : Requires v1.4.0+
- * These functions give the _current_ memory usage of selected object.
- * Note that object memory usage can evolve (increase or decrease) over time. */
-ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
-ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
-ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
-ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
-
-#endif /* ZSTD_H_235446 */
-
-
-/* **************************************************************************************
- * ADVANCED AND EXPERIMENTAL FUNCTIONS
- ****************************************************************************************
- * The definitions in the following section are considered experimental.
- * They are provided for advanced scenarios.
- * They should never be used with a dynamic library, as prototypes may change in the future.
- * Use them only in association with static linking.
- * ***************************************************************************************/
-
-#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
-#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
-
-/* This can be overridden externally to hide static symbols. */
-#ifndef ZSTDLIB_STATIC_API
-# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE
-# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE
-# else
-# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE
-# endif
-#endif
-
-/* Deprecation warnings :
- * Should these warnings be a problem, it is generally possible to disable them,
- * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
- * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
- */
-#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
-# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API /* disable deprecation warnings */
-#else
-# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
-# define ZSTD_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_STATIC_API
-# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
-# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated(message)))
-# elif defined(__GNUC__) && (__GNUC__ >= 3)
-# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated))
-# elif defined(_MSC_VER)
-# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __declspec(deprecated(message))
-# else
-# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
-# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API
-# endif
-#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
-
-/****************************************************************************************
- * experimental API (static linking only)
- ****************************************************************************************
- * The following symbols and constants
- * are not planned to join "stable API" status in the near future.
- * They can still change in future versions.
- * Some of them are planned to remain in the static_only section indefinitely.
- * Some of them might be removed in the future (especially when redundant with existing stable functions)
- * ***************************************************************************************/
-
-#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */
-#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2)
-#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
-#define ZSTD_SKIPPABLEHEADERSIZE 8
-
-/* compression parameter bounds */
-#define ZSTD_WINDOWLOG_MAX_32 30
-#define ZSTD_WINDOWLOG_MAX_64 31
-#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
-#define ZSTD_WINDOWLOG_MIN 10
-#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)
-#define ZSTD_HASHLOG_MIN 6
-#define ZSTD_CHAINLOG_MAX_32 29
-#define ZSTD_CHAINLOG_MAX_64 30
-#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))
-#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
-#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
-#define ZSTD_SEARCHLOG_MIN 1
-#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
-#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */
-#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
-#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
-#define ZSTD_STRATEGY_MIN ZSTD_fast
-#define ZSTD_STRATEGY_MAX ZSTD_btultra2
-
-
-#define ZSTD_OVERLAPLOG_MIN 0
-#define ZSTD_OVERLAPLOG_MAX 9
-
-#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame
- * requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size,
- * to preserve host's memory from unreasonable requirements.
- * This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).
- * The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */
-
-
-/* LDM parameter bounds */
-#define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN
-#define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX
-#define ZSTD_LDM_MINMATCH_MIN 4
-#define ZSTD_LDM_MINMATCH_MAX 4096
-#define ZSTD_LDM_BUCKETSIZELOG_MIN 1
-#define ZSTD_LDM_BUCKETSIZELOG_MAX 8
-#define ZSTD_LDM_HASHRATELOG_MIN 0
-#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
-
-/* Advanced parameter bounds */
-#define ZSTD_TARGETCBLOCKSIZE_MIN 64
-#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
-#define ZSTD_SRCSIZEHINT_MIN 0
-#define ZSTD_SRCSIZEHINT_MAX INT_MAX
-
-
-/* --- Advanced types --- */
-
-typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
-
-typedef struct {
- unsigned int offset; /* The offset of the match. (NOT the same as the offset code)
- * If offset == 0 and matchLength == 0, this sequence represents the last
- * literals in the block of litLength size.
- */
-
- unsigned int litLength; /* Literal length of the sequence. */
- unsigned int matchLength; /* Match length of the sequence. */
-
- /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
- * In this case, we will treat the sequence as a marker for a block boundary.
- */
-
- unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'.
- * Ranges from [0, 3].
- *
- * Repeat offsets are essentially previous offsets from previous sequences sorted in
- * recency order. For more detail, see doc/zstd_compression_format.md
- *
- * If rep == 0, then 'offset' does not contain a repeat offset.
- * If rep > 0:
- * If litLength != 0:
- * rep == 1 --> offset == repeat_offset_1
- * rep == 2 --> offset == repeat_offset_2
- * rep == 3 --> offset == repeat_offset_3
- * If litLength == 0:
- * rep == 1 --> offset == repeat_offset_2
- * rep == 2 --> offset == repeat_offset_3
- * rep == 3 --> offset == repeat_offset_1 - 1
- *
- * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
- * 'rep', but repeat offsets do not necessarily need to be calculated from an external
- * sequence provider's perspective. For example, ZSTD_compressSequences() does not
- * use this 'rep' field at all (as of now).
- */
-} ZSTD_Sequence;
-
-typedef struct {
- unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
- unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
- unsigned hashLog; /**< dispatch table : larger == faster, more memory */
- unsigned searchLog; /**< nb of searches : larger == more compression, slower */
- unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */
- unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
- ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */
-} ZSTD_compressionParameters;
-
-typedef struct {
- int contentSizeFlag; /**< 1: content size will be in frame header (when known) */
- int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */
- int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */
-} ZSTD_frameParameters;
-
-typedef struct {
- ZSTD_compressionParameters cParams;
- ZSTD_frameParameters fParams;
-} ZSTD_parameters;
-
-typedef enum {
- ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
- ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
- ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */
-} ZSTD_dictContentType_e;
-
-typedef enum {
- ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
- ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
-} ZSTD_dictLoadMethod_e;
-
-typedef enum {
- ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
- ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
- * Useful to save 4 bytes per generated frame.
- * Decoder cannot recognise automatically this format, requiring this instruction. */
-} ZSTD_format_e;
-
-typedef enum {
- /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
- ZSTD_d_validateChecksum = 0,
- ZSTD_d_ignoreChecksum = 1
-} ZSTD_forceIgnoreChecksum_e;
-
-typedef enum {
- /* Note: this enum controls ZSTD_d_refMultipleDDicts */
- ZSTD_rmd_refSingleDDict = 0,
- ZSTD_rmd_refMultipleDDicts = 1
-} ZSTD_refMultipleDDicts_e;
-
-typedef enum {
- /* Note: this enum and the behavior it controls are effectively internal
- * implementation details of the compressor. They are expected to continue
- * to evolve and should be considered only in the context of extremely
- * advanced performance tuning.
- *
- * Zstd currently supports the use of a CDict in three ways:
- *
- * - The contents of the CDict can be copied into the working context. This
- * means that the compression can search both the dictionary and input
- * while operating on a single set of internal tables. This makes
- * the compression faster per-byte of input. However, the initial copy of
- * the CDict's tables incurs a fixed cost at the beginning of the
- * compression. For small compressions (< 8 KB), that copy can dominate
- * the cost of the compression.
- *
- * - The CDict's tables can be used in-place. In this model, compression is
- * slower per input byte, because the compressor has to search two sets of
- * tables. However, this model incurs no start-up cost (as long as the
- * working context's tables can be reused). For small inputs, this can be
- * faster than copying the CDict's tables.
- *
- * - The CDict's tables are not used at all, and instead we use the working
- * context alone to reload the dictionary and use params based on the source
- * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
- * This method is effective when the dictionary sizes are very small relative
- * to the input size, and the input size is fairly large to begin with.
- *
- * Zstd has a simple internal heuristic that selects which strategy to use
- * at the beginning of a compression. However, if experimentation shows that
- * Zstd is making poor choices, it is possible to override that choice with
- * this enum.
- */
- ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
- ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
- ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
- ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
-} ZSTD_dictAttachPref_e;
-
-typedef enum {
- ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level.
- * Negative compression levels will be uncompressed, and positive compression
- * levels will be compressed. */
- ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
- * emitted if Huffman compression is not profitable. */
- ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
-} ZSTD_literalCompressionMode_e;
-
-typedef enum {
- /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
- * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
- * or ZSTD_ps_disable allow for a force enable/disable the feature.
- */
- ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */
- ZSTD_ps_enable = 1, /* Force-enable the feature */
- ZSTD_ps_disable = 2 /* Do not use the feature */
-} ZSTD_paramSwitch_e;
-
-/***************************************
-* Frame size functions
-***************************************/
-
-/*! ZSTD_findDecompressedSize() :
- * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
- * `srcSize` must be the _exact_ size of this series
- * (i.e. there should be a frame boundary at `src + srcSize`)
- * @return : - decompressed size of all data in all successive frames
- * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
- * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
- *
- * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
- * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
- * In which case, it's necessary to use streaming mode to decompress data.
- * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
- * note 3 : decompressed size can be very large (64-bits value),
- * potentially larger than what local system can handle as a single memory segment.
- * In which case, it's necessary to use streaming mode to decompress data.
- * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
- * Always ensure result fits within application's authorized limits.
- * Each application can set its own limits.
- * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
- * read each contained frame header. This is fast as most of the data is skipped,
- * however it does mean that all frame data must be present and valid. */
-ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
-
-/*! ZSTD_decompressBound() :
- * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
- * `srcSize` must be the _exact_ size of this series
- * (i.e. there should be a frame boundary at `src + srcSize`)
- * @return : - upper-bound for the decompressed size of all data in all successive frames
- * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
- *
- * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
- * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
- * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
- * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
- * upper-bound = # blocks * min(128 KB, Window_Size)
- */
-ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
-
-/*! ZSTD_frameHeaderSize() :
- * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
- * @return : size of the Frame Header,
- * or an error code (if srcSize is too small) */
-ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
-
-typedef enum {
- ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
- ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
-} ZSTD_sequenceFormat_e;
-
-/*! ZSTD_generateSequences() :
- * Generate sequences using ZSTD_compress2, given a source buffer.
- *
- * Each block will end with a dummy sequence
- * with offset == 0, matchLength == 0, and litLength == length of last literals.
- * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
- * simply acts as a block delimiter.
- *
- * zc can be used to insert custom compression params.
- * This function invokes ZSTD_compress2
- *
- * The output of this function can be fed into ZSTD_compressSequences() with CCtx
- * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
- * @return : number of sequences generated
- */
-
-ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
- size_t outSeqsSize, const void* src, size_t srcSize);
-
-/*! ZSTD_mergeBlockDelimiters() :
- * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
- * by merging them into into the literals of the next sequence.
- *
- * As such, the final generated result has no explicit representation of block boundaries,
- * and the final last literals segment is not represented in the sequences.
- *
- * The output of this function can be fed into ZSTD_compressSequences() with CCtx
- * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
- * @return : number of sequences left after merging
- */
-ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
-
-/*! ZSTD_compressSequences() :
- * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
- * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
- * The entire source is compressed into a single frame.
- *
- * The compression behavior changes based on cctx params. In particular:
- * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
- * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
- * the block size derived from the cctx, and sequences may be split. This is the default setting.
- *
- * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
- * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
- *
- * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
- * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
- * specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
- *
- * In addition to the two adjustable experimental params, there are other important cctx params.
- * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
- * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
- * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
- * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
- *
- * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
- * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
- * and cannot emit an RLE block that disagrees with the repcode history
- * @return : final compressed size or a ZSTD error.
- */
-ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
- const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
- const void* src, size_t srcSize);
-
-
-/*! ZSTD_writeSkippableFrame() :
- * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
- *
- * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,
- * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
- * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
- * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
- *
- * Returns an error if destination buffer is not large enough, if the source size is not representable
- * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
- *
- * @return : number of bytes written or a ZSTD error.
- */
-ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
- const void* src, size_t srcSize, unsigned magicVariant);
-
-/*! ZSTD_readSkippableFrame() :
- * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
- *
- * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
- * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
- * in the magicVariant.
- *
- * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
- *
- * @return : number of bytes written or a ZSTD error.
- */
-ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
- const void* src, size_t srcSize);
-
-/*! ZSTD_isSkippableFrame() :
- * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
- */
-ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
-
-
-
-/***************************************
-* Memory management
-***************************************/
-
-/*! ZSTD_estimate*() :
- * These functions make it possible to estimate memory usage
- * of a future {D,C}Ctx, before its creation.
- *
- * ZSTD_estimateCCtxSize() will provide a memory budget large enough
- * for any compression level up to selected one.
- * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate
- * does not include space for a window buffer.
- * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
- * The estimate will assume the input may be arbitrarily large,
- * which is the worst case.
- *
- * When srcSize can be bound by a known and rather "small" value,
- * this fact can be used to provide a tighter estimation
- * because the CCtx compression context will need less memory.
- * This tighter estimation can be provided by more advanced functions
- * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
- * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
- * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
- *
- * Note 2 : only single-threaded compression is supported.
- * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
- */
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
-
-/*! ZSTD_estimateCStreamSize() :
- * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
- * It will also consider src size to be arbitrarily "large", which is worst case.
- * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
- * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
- * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
- * Note : CStream size estimation is only correct for single-threaded compression.
- * ZSTD_DStream memory budget depends on window Size.
- * This information can be passed manually, using ZSTD_estimateDStreamSize,
- * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
- * Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
- * an internal ?Dict will be created, which additional size is not estimated here.
- * In this case, get total size by adding ZSTD_estimate?DictSize */
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
-
-/*! ZSTD_estimate?DictSize() :
- * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
- * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
- * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
- */
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
-
-/*! ZSTD_initStatic*() :
- * Initialize an object using a pre-allocated fixed-size buffer.
- * workspace: The memory area to emplace the object into.
- * Provided pointer *must be 8-bytes aligned*.
- * Buffer must outlive object.
- * workspaceSize: Use ZSTD_estimate*Size() to determine
- * how large workspace must be to support target scenario.
- * @return : pointer to object (same address as workspace, just different type),
- * or NULL if error (size too small, incorrect alignment, etc.)
- * Note : zstd will never resize nor malloc() when using a static buffer.
- * If the object requires more memory than available,
- * zstd will just error out (typically ZSTD_error_memory_allocation).
- * Note 2 : there is no corresponding "free" function.
- * Since workspace is allocated externally, it must be freed externally too.
- * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level
- * into its associated cParams.
- * Limitation 1 : currently not compatible with internal dictionary creation, triggered by
- * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict().
- * Limitation 2 : static cctx currently not compatible with multi-threading.
- * Limitation 3 : static dctx is incompatible with legacy support.
- */
-ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
-ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
-
-ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
-ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
-
-ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict(
- void* workspace, size_t workspaceSize,
- const void* dict, size_t dictSize,
- ZSTD_dictLoadMethod_e dictLoadMethod,
- ZSTD_dictContentType_e dictContentType,
- ZSTD_compressionParameters cParams);
-
-ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict(
- void* workspace, size_t workspaceSize,
- const void* dict, size_t dictSize,
- ZSTD_dictLoadMethod_e dictLoadMethod,
- ZSTD_dictContentType_e dictContentType);
-
-
-/*! Custom memory allocation :
- * These prototypes make it possible to pass your own allocation/free functions.
- * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
- * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
- */
-typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
-typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
-typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
-static
-#ifdef __GNUC__
-__attribute__((__unused__))
-#endif
-ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
-
-ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
-ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
-ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
-ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
-
-ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
- ZSTD_dictLoadMethod_e dictLoadMethod,
- ZSTD_dictContentType_e dictContentType,
- ZSTD_compressionParameters cParams,
- ZSTD_customMem customMem);
-
-/*! Thread pool :
- * These prototypes make it possible to share a thread pool among multiple compression contexts.
- * This can limit resources for applications with multiple threads where each one uses
- * a threaded compression mode (via ZSTD_c_nbWorkers parameter).
- * ZSTD_createThreadPool creates a new thread pool with a given number of threads.
- * Note that the lifetime of such pool must exist while being used.
- * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
- * to use an internal thread pool).
- * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer.
- */
-typedef struct POOL_ctx_s ZSTD_threadPool;
-ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
-ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */
-ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
-
-
-/*
- * This API is temporary and is expected to change or disappear in the future!
- */
-ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2(
- const void* dict, size_t dictSize,
- ZSTD_dictLoadMethod_e dictLoadMethod,
- ZSTD_dictContentType_e dictContentType,
- const ZSTD_CCtx_params* cctxParams,
- ZSTD_customMem customMem);
-
-ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced(
- const void* dict, size_t dictSize,
- ZSTD_dictLoadMethod_e dictLoadMethod,
- ZSTD_dictContentType_e dictContentType,
- ZSTD_customMem customMem);
-
-
-/***************************************
-* Advanced compression functions
-***************************************/
-
-/*! ZSTD_createCDict_byReference() :
- * Create a digested dictionary for compression
- * Dictionary content is just referenced, not duplicated.
- * As a consequence, `dictBuffer` **must** outlive CDict,
- * and its content must remain unmodified throughout the lifetime of CDict.
- * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
-ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
-
-/*! ZSTD_getCParams() :
- * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
- * `estimatedSrcSize` value is optional, select 0 if not known */
-ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
-
-/*! ZSTD_getParams() :
- * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
- * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
-ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
-
-/*! ZSTD_checkCParams() :
- * Ensure param values remain within authorized range.
- * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
-ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
-
-/*! ZSTD_adjustCParams() :
- * optimize params for a given `srcSize` and `dictSize`.
- * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
- * `dictSize` must be `0` when there is no dictionary.
- * cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
- * This function never fails (wide contract) */
-ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
-
-/*! ZSTD_compress_advanced() :
- * Note : this function is now DEPRECATED.
- * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
- * This prototype will generate compilation warnings. */
-ZSTD_DEPRECATED("use ZSTD_compress2")
-size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize,
- ZSTD_parameters params);
-
-/*! ZSTD_compress_usingCDict_advanced() :
- * Note : this function is now DEPRECATED.
- * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
- * This prototype will generate compilation warnings. */
-ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
-size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTD_CDict* cdict,
- ZSTD_frameParameters fParams);
-
-
-/*! ZSTD_CCtx_loadDictionary_byReference() :
- * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
- * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
-ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
-
-/*! ZSTD_CCtx_loadDictionary_advanced() :
- * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
- * how to load the dictionary (by copy ? by reference ?)
- * and how to interpret it (automatic ? force raw mode ? full mode only ?) */
-ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
-
-/*! ZSTD_CCtx_refPrefix_advanced() :
- * Same as ZSTD_CCtx_refPrefix(), but gives finer control over
- * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
-ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
-
-/* === experimental parameters === */
-/* these parameters can be used with ZSTD_setParameter()
- * they are not guaranteed to remain supported in the future */
-
- /* Enables rsyncable mode,
- * which makes compressed files more rsync friendly
- * by adding periodic synchronization points to the compressed data.
- * The target average block size is ZSTD_c_jobSize / 2.
- * It's possible to modify the job size to increase or decrease
- * the granularity of the synchronization point.
- * Once the jobSize is smaller than the window size,
- * it will result in compression ratio degradation.
- * NOTE 1: rsyncable mode only works when multithreading is enabled.
- * NOTE 2: rsyncable performs poorly in combination with long range mode,
- * since it will decrease the effectiveness of synchronization points,
- * though mileage may vary.
- * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s.
- * If the selected compression level is already running significantly slower,
- * the overall speed won't be significantly impacted.
- */
- #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1
-
-/* Select a compression format.
- * The value must be of type ZSTD_format_e.
- * See ZSTD_format_e enum definition for details */
-#define ZSTD_c_format ZSTD_c_experimentalParam2
-
-/* Force back-reference distances to remain < windowSize,
- * even when referencing into Dictionary content (default:0) */
-#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3
-
-/* Controls whether the contents of a CDict
- * are used in place, or copied into the working context.
- * Accepts values from the ZSTD_dictAttachPref_e enum.
- * See the comments on that enum for an explanation of the feature. */
-#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
-
-/* Controlled with ZSTD_paramSwitch_e enum.
- * Default is ZSTD_ps_auto.
- * Set to ZSTD_ps_disable to never compress literals.
- * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
- * may still be emitted if huffman is not beneficial to use.)
- *
- * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
- * literals compression based on the compression parameters - specifically,
- * negative compression levels do not use literal compression.
- */
-#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
-
-/* Tries to fit compressed block size to be around targetCBlockSize.
- * No target when targetCBlockSize == 0.
- * There is no guarantee on compressed block size (default:0) */
-#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6
-
-/* User's best guess of source size.
- * Hint is not valid when srcSizeHint == 0.
- * There is no guarantee that hint is close to actual source size,
- * but compression ratio may regress significantly if guess considerably underestimates */
-#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
-
-/* Controls whether the new and experimental "dedicated dictionary search
- * structure" can be used. This feature is still rough around the edges, be
- * prepared for surprising behavior!
- *
- * How to use it:
- *
- * When using a CDict, whether to use this feature or not is controlled at
- * CDict creation, and it must be set in a CCtxParams set passed into that
- * construction (via ZSTD_createCDict_advanced2()). A compression will then
- * use the feature or not based on how the CDict was constructed; the value of
- * this param, set in the CCtx, will have no effect.
- *
- * However, when a dictionary buffer is passed into a CCtx, such as via
- * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control
- * whether the CDict that is created internally can use the feature or not.
- *
- * What it does:
- *
- * Normally, the internal data structures of the CDict are analogous to what
- * would be stored in a CCtx after compressing the contents of a dictionary.
- * To an approximation, a compression using a dictionary can then use those
- * data structures to simply continue what is effectively a streaming
- * compression where the simulated compression of the dictionary left off.
- * Which is to say, the search structures in the CDict are normally the same
- * format as in the CCtx.
- *
- * It is possible to do better, since the CDict is not like a CCtx: the search
- * structures are written once during CDict creation, and then are only read
- * after that, while the search structures in the CCtx are both read and
- * written as the compression goes along. This means we can choose a search
- * structure for the dictionary that is read-optimized.
- *
- * This feature enables the use of that different structure.
- *
- * Note that some of the members of the ZSTD_compressionParameters struct have
- * different semantics and constraints in the dedicated search structure. It is
- * highly recommended that you simply set a compression level in the CCtxParams
- * you pass into the CDict creation call, and avoid messing with the cParams
- * directly.
- *
- * Effects:
- *
- * This will only have any effect when the selected ZSTD_strategy
- * implementation supports this feature. Currently, that's limited to
- * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2.
- *
- * Note that this means that the CDict tables can no longer be copied into the
- * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
- * usable. The dictionary can only be attached or reloaded.
- *
- * In general, you should expect compression to be faster--sometimes very much
- * so--and CDict creation to be slightly slower. Eventually, we will probably
- * make this mode the default.
- */
-#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
-
-/* ZSTD_c_stableInBuffer
- * Experimental parameter.
- * Default is 0 == disabled. Set to 1 to enable.
- *
- * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same
- * between calls, except for the modifications that zstd makes to pos (the
- * caller must not modify pos). This is checked by the compressor, and
- * compression will fail if it ever changes. This means the only flush
- * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end
- * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos)
- * MUST not be modified during compression or you will get data corruption.
- *
- * When this flag is enabled zstd won't allocate an input window buffer,
- * because the user guarantees it can reference the ZSTD_inBuffer until
- * the frame is complete. But, it will still allocate an output buffer
- * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
- * avoid the memcpy() from the input buffer to the input window buffer.
- *
- * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
- * That means this flag cannot be used with ZSTD_compressStream().
- *
- * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
- * this flag is ALWAYS memory safe, and will never access out-of-bounds
- * memory. However, compression WILL fail if you violate the preconditions.
- *
- * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST
- * not be modified during compression or you will get data corruption. This
- * is because zstd needs to reference data in the ZSTD_inBuffer to find
- * matches. Normally zstd maintains its own window buffer for this purpose,
- * but passing this flag tells zstd to use the user provided buffer.
- */
-#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
-
-/* ZSTD_c_stableOutBuffer
- * Experimental parameter.
- * Default is 0 == disabled. Set to 1 to enable.
- *
- * Tells he compressor that the ZSTD_outBuffer will not be resized between
- * calls. Specifically: (out.size - out.pos) will never grow. This gives the
- * compressor the freedom to say: If the compressed data doesn't fit in the
- * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
- * always decompress directly into the output buffer, instead of decompressing
- * into an internal buffer and copying to the output buffer.
- *
- * When this flag is enabled zstd won't allocate an output buffer, because
- * it can write directly to the ZSTD_outBuffer. It will still allocate the
- * input window buffer (see ZSTD_c_stableInBuffer).
- *
- * Zstd will check that (out.size - out.pos) never grows and return an error
- * if it does. While not strictly necessary, this should prevent surprises.
- */
-#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
-
-/* ZSTD_c_blockDelimiters
- * Default is 0 == ZSTD_sf_noBlockDelimiters.
- *
- * For use with sequence compression API: ZSTD_compressSequences().
- *
- * Designates whether or not the given array of ZSTD_Sequence contains block delimiters
- * and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
- * See the definition of ZSTD_Sequence for more specifics.
- */
-#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11
-
-/* ZSTD_c_validateSequences
- * Default is 0 == disabled. Set to 1 to enable sequence validation.
- *
- * For use with sequence compression API: ZSTD_compressSequences().
- * Designates whether or not we validate sequences provided to ZSTD_compressSequences()
- * during function execution.
- *
- * Without validation, providing a sequence that does not conform to the zstd spec will cause
- * undefined behavior, and may produce a corrupted block.
- *
- * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for
- * specifics regarding offset/matchlength requirements) then the function will bail out and
- * return an error.
- *
- */
-#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
-
-/* ZSTD_c_useBlockSplitter
- * Controlled with ZSTD_paramSwitch_e enum.
- * Default is ZSTD_ps_auto.
- * Set to ZSTD_ps_disable to never use block splitter.
- * Set to ZSTD_ps_enable to always use block splitter.
- *
- * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
- * block splitting based on the compression parameters.
- */
-#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13
-
-/* ZSTD_c_useRowMatchFinder
- * Controlled with ZSTD_paramSwitch_e enum.
- * Default is ZSTD_ps_auto.
- * Set to ZSTD_ps_disable to never use row-based matchfinder.
- * Set to ZSTD_ps_enable to force usage of row-based matchfinder.
- *
- * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
- * the row-based matchfinder based on support for SIMD instructions and the window log.
- * Note that this only pertains to compression strategies: greedy, lazy, and lazy2
- */
-#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
-
-/* ZSTD_c_deterministicRefPrefix
- * Default is 0 == disabled. Set to 1 to enable.
- *
- * Zstd produces different results for prefix compression when the prefix is
- * directly adjacent to the data about to be compressed vs. when it isn't.
- * This is because zstd detects that the two buffers are contiguous and it can
- * use a more efficient match finding algorithm. However, this produces different
- * results than when the two buffers are non-contiguous. This flag forces zstd
- * to always load the prefix in non-contiguous mode, even if it happens to be
- * adjacent to the data, to guarantee determinism.
- *
- * If you really care about determinism when using a dictionary or prefix,
- * like when doing delta compression, you should select this option. It comes
- * at a speed penalty of about ~2.5% if the dictionary and data happened to be
- * contiguous, and is free if they weren't contiguous. We don't expect that
- * intentionally making the dictionary and data contiguous will be worth the
- * cost to memcpy() the data.
- */
-#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
-
-/*! ZSTD_CCtx_getParameter() :
- * Get the requested compression parameter value, selected by enum ZSTD_cParameter,
- * and store it into int* value.
- * @return : 0, or an error code (which can be tested with ZSTD_isError()).
- */
-ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
-
-
-/*! ZSTD_CCtx_params :
- * Quick howto :
- * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
- * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
- * an existing ZSTD_CCtx_params structure.
- * This is similar to
- * ZSTD_CCtx_setParameter().
- * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
- * an existing CCtx.
- * These parameters will be applied to
- * all subsequent frames.
- * - ZSTD_compressStream2() : Do compression using the CCtx.
- * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer.
- *
- * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
- * for static allocation of CCtx for single-threaded compression.
- */
-ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
-ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */
-
-/*! ZSTD_CCtxParams_reset() :
- * Reset params to default values.
- */
-ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
-
-/*! ZSTD_CCtxParams_init() :
- * Initializes the compression parameters of cctxParams according to
- * compression level. All other parameters are reset to their default values.
- */
-ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
-
-/*! ZSTD_CCtxParams_init_advanced() :
- * Initializes the compression and frame parameters of cctxParams according to
- * params. All other parameters are reset to their default values.
- */
-ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
-
-/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+
- * Similar to ZSTD_CCtx_setParameter.
- * Set one compression parameter, selected by enum ZSTD_cParameter.
- * Parameters must be applied to a ZSTD_CCtx using
- * ZSTD_CCtx_setParametersUsingCCtxParams().
- * @result : a code representing success or failure (which can be tested with
- * ZSTD_isError()).
- */
-ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
-
-/*! ZSTD_CCtxParams_getParameter() :
- * Similar to ZSTD_CCtx_getParameter.
- * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- */
-ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
-
-/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
- * Apply a set of ZSTD_CCtx_params to the compression context.
- * This can be done even after compression is started,
- * if nbWorkers==0, this will have no impact until a new compression is started.
- * if nbWorkers>=1, new parameters will be picked up at next job,
- * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
- */
-ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
- ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
-
-/*! ZSTD_compressStream2_simpleArgs() :
- * Same as ZSTD_compressStream2(),
- * but using only integral types as arguments.
- * This variant might be helpful for binders from dynamic languages
- * which have troubles handling structures containing memory pointers.
- */
-ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs (
- ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity, size_t* dstPos,
- const void* src, size_t srcSize, size_t* srcPos,
- ZSTD_EndDirective endOp);
-
-
-/***************************************
-* Advanced decompression functions
-***************************************/
-
-/*! ZSTD_isFrame() :
- * Tells if the content of `buffer` starts with a valid Frame Identifier.
- * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
- * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
- * Note 3 : Skippable Frame Identifiers are considered valid. */
-ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
-
-/*! ZSTD_createDDict_byReference() :
- * Create a digested dictionary, ready to start decompression operation without startup delay.
- * Dictionary content is referenced, and therefore stays in dictBuffer.
- * It is important that dictBuffer outlives DDict,
- * it must remain read accessible throughout the lifetime of DDict */
-ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
-
-/*! ZSTD_DCtx_loadDictionary_byReference() :
- * Same as ZSTD_DCtx_loadDictionary(),
- * but references `dict` content instead of copying it into `dctx`.
- * This saves memory if `dict` remains around.,
- * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
-ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-
-/*! ZSTD_DCtx_loadDictionary_advanced() :
- * Same as ZSTD_DCtx_loadDictionary(),
- * but gives direct control over
- * how to load the dictionary (by copy ? by reference ?)
- * and how to interpret it (automatic ? force raw mode ? full mode only ?). */
-ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
-
-/*! ZSTD_DCtx_refPrefix_advanced() :
- * Same as ZSTD_DCtx_refPrefix(), but gives finer control over
- * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
-ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
-
-/*! ZSTD_DCtx_setMaxWindowSize() :
- * Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
- * This protects a decoder context from reserving too much memory for itself (potential attack scenario).
- * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
- * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
- * @return : 0, or an error code (which can be tested using ZSTD_isError()).
- */
-ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
-
-/*! ZSTD_DCtx_getParameter() :
- * Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
- * and store it into int* value.
- * @return : 0, or an error code (which can be tested with ZSTD_isError()).
- */
-ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
-
-/* ZSTD_d_format
- * experimental parameter,
- * allowing selection between ZSTD_format_e input compression formats
- */
-#define ZSTD_d_format ZSTD_d_experimentalParam1
-/* ZSTD_d_stableOutBuffer
- * Experimental parameter.
- * Default is 0 == disabled. Set to 1 to enable.
- *
- * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same
- * between calls, except for the modifications that zstd makes to pos (the
- * caller must not modify pos). This is checked by the decompressor, and
- * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer
- * MUST be large enough to fit the entire decompressed frame. This will be
- * checked when the frame content size is known. The data in the ZSTD_outBuffer
- * in the range [dst, dst + pos) MUST not be modified during decompression
- * or you will get data corruption.
- *
- * When this flags is enabled zstd won't allocate an output buffer, because
- * it can write directly to the ZSTD_outBuffer, but it will still allocate
- * an input buffer large enough to fit any compressed block. This will also
- * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
- * If you need to avoid the input buffer allocation use the buffer-less
- * streaming API.
- *
- * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using
- * this flag is ALWAYS memory safe, and will never access out-of-bounds
- * memory. However, decompression WILL fail if you violate the preconditions.
- *
- * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST
- * not be modified during decompression or you will get data corruption. This
- * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate
- * matches. Normally zstd maintains its own buffer for this purpose, but passing
- * this flag tells zstd to use the user provided buffer.
- */
-#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2
-
-/* ZSTD_d_forceIgnoreChecksum
- * Experimental parameter.
- * Default is 0 == disabled. Set to 1 to enable
- *
- * Tells the decompressor to skip checksum validation during decompression, regardless
- * of whether checksumming was specified during compression. This offers some
- * slight performance benefits, and may be useful for debugging.
- * Param has values of type ZSTD_forceIgnoreChecksum_e
- */
-#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
-
-/* ZSTD_d_refMultipleDDicts
- * Experimental parameter.
- * Default is 0 == disabled. Set to 1 to enable
- *
- * If enabled and dctx is allocated on the heap, then additional memory will be allocated
- * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
- * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
- * store all references. At decompression time, the appropriate dictID is selected
- * from the set of DDicts based on the dictID in the frame.
- *
- * Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
- *
- * Param has values of byte ZSTD_refMultipleDDicts_e
- *
- * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
- * allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
- * Memory is allocated as per ZSTD_DCtx::customMem.
- *
- * Although this function allocates memory for the table, the user is still responsible for
- * memory management of the underlying ZSTD_DDict* themselves.
- */
-#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
-
-
-/*! ZSTD_DCtx_setFormat() :
- * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
- * Instruct the decoder context about what kind of data to decode next.
- * This instruction is mandatory to decode data without a fully-formed header,
- * such ZSTD_f_zstd1_magicless for example.
- * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
-ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
-size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
-
-/*! ZSTD_decompressStream_simpleArgs() :
- * Same as ZSTD_decompressStream(),
- * but using only integral types as arguments.
- * This can be helpful for binders from dynamic languages
- * which have troubles handling structures containing memory pointers.
- */
-ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
- ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity, size_t* dstPos,
- const void* src, size_t srcSize, size_t* srcPos);
-
-
-/********************************************************************
-* Advanced streaming functions
-* Warning : most of these functions are now redundant with the Advanced API.
-* Once Advanced API reaches "stable" status,
-* redundant functions will be deprecated, and then at some point removed.
-********************************************************************/
-
-/*===== Advanced Streaming compression functions =====*/
-
-/*! ZSTD_initCStream_srcSize() :
- * This function is DEPRECATED, and equivalent to:
- * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
- * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
- * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
- *
- * pledgedSrcSize must be correct. If it is not known at init time, use
- * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
- * "0" also disables frame content size field. It may be enabled in the future.
- * This prototype will generate compilation warnings.
- */
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
-size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
- int compressionLevel,
- unsigned long long pledgedSrcSize);
-
-/*! ZSTD_initCStream_usingDict() :
- * This function is DEPRECATED, and is equivalent to:
- * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
- * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
- *
- * Creates of an internal CDict (incompatible with static CCtx), except if
- * dict == NULL or dictSize < 8, in which case no dict is used.
- * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
- * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
- * This prototype will generate compilation warnings.
- */
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
- const void* dict, size_t dictSize,
- int compressionLevel);
-
-/*! ZSTD_initCStream_advanced() :
- * This function is DEPRECATED, and is approximately equivalent to:
- * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- * // Pseudocode: Set each zstd parameter and leave the rest as-is.
- * for ((param, value) : params) {
- * ZSTD_CCtx_setParameter(zcs, param, value);
- * }
- * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
- * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
- *
- * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
- * pledgedSrcSize must be correct.
- * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
- * This prototype will generate compilation warnings.
- */
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
-size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
- const void* dict, size_t dictSize,
- ZSTD_parameters params,
- unsigned long long pledgedSrcSize);
-
-/*! ZSTD_initCStream_usingCDict() :
- * This function is DEPRECATED, and equivalent to:
- * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- * ZSTD_CCtx_refCDict(zcs, cdict);
- *
- * note : cdict will just be referenced, and must outlive compression session
- * This prototype will generate compilation warnings.
- */
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
-size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
-
-/*! ZSTD_initCStream_usingCDict_advanced() :
- * This function is DEPRECATED, and is approximately equivalent to:
- * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- * // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
- * for ((fParam, value) : fParams) {
- * ZSTD_CCtx_setParameter(zcs, fParam, value);
- * }
- * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
- * ZSTD_CCtx_refCDict(zcs, cdict);
- *
- * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
- * pledgedSrcSize must be correct. If srcSize is not known at init time, use
- * value ZSTD_CONTENTSIZE_UNKNOWN.
- * This prototype will generate compilation warnings.
- */
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
-size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
- const ZSTD_CDict* cdict,
- ZSTD_frameParameters fParams,
- unsigned long long pledgedSrcSize);
-
-/*! ZSTD_resetCStream() :
- * This function is DEPRECATED, and is equivalent to:
- * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
- * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
- * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
- * explicitly specified.
- *
- * start a new frame, using same parameters from previous frame.
- * This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
- * Note that zcs must be init at least once before using ZSTD_resetCStream().
- * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
- * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
- * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
- * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
- * @return : 0, or an error code (which can be tested using ZSTD_isError())
- * This prototype will generate compilation warnings.
- */
-ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
-size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
-
-
-typedef struct {
- unsigned long long ingested; /* nb input bytes read and buffered */
- unsigned long long consumed; /* nb input bytes actually compressed */
- unsigned long long produced; /* nb of compressed bytes generated and buffered */
- unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
- unsigned currentJobID; /* MT only : latest started job nb */
- unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */
-} ZSTD_frameProgression;
-
-/* ZSTD_getFrameProgression() :
- * tells how much data has been ingested (read from input)
- * consumed (input actually compressed) and produced (output) for current frame.
- * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
- * Aggregates progression inside active worker threads.
- */
-ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
-
-/*! ZSTD_toFlushNow() :
- * Tell how many bytes are ready to be flushed immediately.
- * Useful for multithreading scenarios (nbWorkers >= 1).
- * Probe the oldest active job, defined as oldest job not yet entirely flushed,
- * and check its output buffer.
- * @return : amount of data stored in oldest job and ready to be flushed immediately.
- * if @return == 0, it means either :
- * + there is no active job (could be checked with ZSTD_frameProgression()), or
- * + oldest job is still actively compressing data,
- * but everything it has produced has also been flushed so far,
- * therefore flush speed is limited by production speed of oldest job
- * irrespective of the speed of concurrent (and newer) jobs.
- */
-ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
-
-
-/*===== Advanced Streaming decompression functions =====*/
-
-/*!
- * This function is deprecated, and is equivalent to:
- *
- * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
- * ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
- *
- * note: no dictionary will be used if dict == NULL or dictSize < 8
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
- */
-ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
-
-/*!
- * This function is deprecated, and is equivalent to:
- *
- * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
- * ZSTD_DCtx_refDDict(zds, ddict);
- *
- * note : ddict is referenced, it must outlive decompression session
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
- */
-ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
-
-/*!
- * This function is deprecated, and is equivalent to:
- *
- * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
- *
- * re-use decompression parameters from previous init; saves dictionary loading
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
- */
-ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
-
-
-/*********************************************************************
-* Buffer-less and synchronous inner streaming functions
-*
-* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
-* But it's also a complex one, with several restrictions, documented below.
-* Prefer normal streaming API for an easier experience.
-********************************************************************* */
-
-/**
- Buffer-less streaming compression (synchronous mode)
-
- A ZSTD_CCtx object is required to track streaming operations.
- Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
- ZSTD_CCtx object can be re-used multiple times within successive compression operations.
-
- Start by initializing a context.
- Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
- It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
-
- Then, consume your input using ZSTD_compressContinue().
- There are some important considerations to keep in mind when using this advanced function :
- - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
- - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
- - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
- Worst case evaluation is provided by ZSTD_compressBound().
- ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
- - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
- It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
- - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
- In which case, it will "discard" the relevant memory section from its history.
-
- Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
- It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
- Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
-
- `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
-*/
-
-/*===== Buffer-less streaming compression functions =====*/
-ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
-ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
-ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
-
-ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-
-/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
-ZSTD_DEPRECATED("use advanced API to access custom parameters")
-size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
-ZSTD_DEPRECATED("use advanced API to access custom parameters")
-size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
-/**
- Buffer-less streaming decompression (synchronous mode)
-
- A ZSTD_DCtx object is required to track streaming operations.
- Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
- A ZSTD_DCtx object can be re-used multiple times.
-
- First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
- Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
- Data fragment must be large enough to ensure successful decoding.
- `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
- @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
- >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
- errorCode, which can be tested using ZSTD_isError().
-
- It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
- such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
- Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
- As a consequence, check that values remain within valid application range.
- For example, do not allocate memory blindly, check that `windowSize` is within expectation.
- Each application can set its own limits, depending on local restrictions.
- For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
-
- ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
- ZSTD_decompressContinue() is very sensitive to contiguity,
- if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
- or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
- There are multiple ways to guarantee this condition.
-
- The most memory efficient way is to use a round buffer of sufficient size.
- Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
- which can @return an error code if required value is too large for current system (in 32-bits mode).
- In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
- up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
- which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
- At which point, decoding can resume from the beginning of the buffer.
- Note that already decoded data stored in the buffer should be flushed before being overwritten.
-
- There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
-
- Finally, if you control the compression process, you can also ignore all buffer size rules,
- as long as the encoder and decoder progress in "lock-step",
- aka use exactly the same buffer sizes, break contiguity at the same place, etc.
-
- Once buffers are setup, start decompression, with ZSTD_decompressBegin().
- If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
-
- Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
- ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
- ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
-
- @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
- It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
- It can also be an error code, which can be tested with ZSTD_isError().
-
- A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
- Context can then be reset to start a new decompression.
-
- Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
- This information is not required to properly decode a frame.
-
- == Special case : skippable frames ==
-
- Skippable frames allow integration of user-defined data into a flow of concatenated frames.
- Skippable frames will be ignored (skipped) by decompressor.
- The format of skippable frames is as follows :
- a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
- b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
- c) Frame Content - any content (User Data) of length equal to Frame Size
- For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
- For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
-*/
-
-/*===== Buffer-less streaming decompression functions =====*/
-typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
-typedef struct {
- unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
- unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
- unsigned blockSizeMax;
- ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
- unsigned headerSize;
- unsigned dictID;
- unsigned checksumFlag;
-} ZSTD_frameHeader;
-
-/*! ZSTD_getFrameHeader() :
- * decode Frame Header, or requires larger `srcSize`.
- * @return : 0, `zfhPtr` is correctly filled,
- * >0, `srcSize` is too small, value is wanted `srcSize` amount,
- * or an error code, which can be tested using ZSTD_isError() */
-ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
-/*! ZSTD_getFrameHeader_advanced() :
- * same as ZSTD_getFrameHeader(),
- * with added capability to select a format (like ZSTD_f_zstd1_magicless) */
-ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
-ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
-
-ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
-ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
-
-ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
-ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-
-/* misc */
-ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
-typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
-ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
-
-
-
-
-/* ============================ */
-/** Block level API */
-/* ============================ */
-
-/*!
- Block functions produce and decode raw zstd blocks, without frame metadata.
- Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
- But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
-
- A few rules to respect :
- - Compressing and decompressing require a context structure
- + Use ZSTD_createCCtx() and ZSTD_createDCtx()
- - It is necessary to init context before starting
- + compression : any ZSTD_compressBegin*() variant, including with dictionary
- + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
- + copyCCtx() and copyDCtx() can be used too
- - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
- + If input is larger than a block size, it's necessary to split input data into multiple blocks
- + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
- Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
- - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
- ===> In which case, nothing is produced into `dst` !
- + User __must__ test for such outcome and deal directly with uncompressed data
- + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
- Doing so would mess up with statistics history, leading to potential data corruption.
- + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
- + In case of multiple successive blocks, should some of them be uncompressed,
- decoder must be informed of their existence in order to follow proper history.
- Use ZSTD_insertBlock() for such a case.
-*/
-
-/*===== Raw zstd block functions =====*/
-ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
-ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
-
-
-#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
-
-#if defined (__cplusplus)
-}
-#endif
stage1/zstd/lib/zstd_errors.h
@@ -1,95 +0,0 @@
-/*
- * Copyright (c) Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef ZSTD_ERRORS_H_398273423
-#define ZSTD_ERRORS_H_398273423
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-/*===== dependency =====*/
-#include <stddef.h> /* size_t */
-
-
-/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
-#ifndef ZSTDERRORLIB_VISIBILITY
-# if defined(__GNUC__) && (__GNUC__ >= 4)
-# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
-# else
-# define ZSTDERRORLIB_VISIBILITY
-# endif
-#endif
-#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
-#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
-#else
-# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
-#endif
-
-/*-*********************************************
- * Error codes list
- *-*********************************************
- * Error codes _values_ are pinned down since v1.3.1 only.
- * Therefore, don't rely on values if you may link to any version < v1.3.1.
- *
- * Only values < 100 are considered stable.
- *
- * note 1 : this API shall be used with static linking only.
- * dynamic linking is not yet officially supported.
- * note 2 : Prefer relying on the enum than on its value whenever possible
- * This is the only supported way to use the error list < v1.3.1
- * note 3 : ZSTD_isError() is always correct, whatever the library version.
- **********************************************/
-typedef enum {
- ZSTD_error_no_error = 0,
- ZSTD_error_GENERIC = 1,
- ZSTD_error_prefix_unknown = 10,
- ZSTD_error_version_unsupported = 12,
- ZSTD_error_frameParameter_unsupported = 14,
- ZSTD_error_frameParameter_windowTooLarge = 16,
- ZSTD_error_corruption_detected = 20,
- ZSTD_error_checksum_wrong = 22,
- ZSTD_error_dictionary_corrupted = 30,
- ZSTD_error_dictionary_wrong = 32,
- ZSTD_error_dictionaryCreation_failed = 34,
- ZSTD_error_parameter_unsupported = 40,
- ZSTD_error_parameter_outOfBound = 42,
- ZSTD_error_tableLog_tooLarge = 44,
- ZSTD_error_maxSymbolValue_tooLarge = 46,
- ZSTD_error_maxSymbolValue_tooSmall = 48,
- ZSTD_error_stage_wrong = 60,
- ZSTD_error_init_missing = 62,
- ZSTD_error_memory_allocation = 64,
- ZSTD_error_workSpace_tooSmall= 66,
- ZSTD_error_dstSize_tooSmall = 70,
- ZSTD_error_srcSize_wrong = 72,
- ZSTD_error_dstBuffer_null = 74,
- /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
- ZSTD_error_frameIndex_tooLarge = 100,
- ZSTD_error_seekableIO = 102,
- ZSTD_error_dstBuffer_wrong = 104,
- ZSTD_error_srcBuffer_wrong = 105,
- ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
-} ZSTD_ErrorCode;
-
-/*! ZSTD_getErrorCode() :
- convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
- which can be used to compare with enum list published above */
-ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
-ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ZSTD_ERRORS_H_398273423 */
stage1/zstd/LICENSE
@@ -1,30 +0,0 @@
-BSD License
-
-For Zstandard software
-
-Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
- * Neither the name Facebook nor the names of its contributors may be used to
- endorse or promote products derived from this software without specific
- prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
stage1/InputStream.h
@@ -4,67 +4,35 @@
#include "panic.h"
#include "wasm.h"
-#include <zstd.h>
-
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <stddef.h>
struct InputStream {
FILE *stream;
- ZSTD_DStream *ds;
- ZSTD_outBuffer out;
- ZSTD_inBuffer in;
- size_t pos;
};
static void InputStream_open(struct InputStream *self, const char *path) {
self->stream = fopen(path, "rb");
if (self->stream == NULL) panic("unable to open input file");
- self->ds = ZSTD_createDStream();
- if (self->ds == NULL) panic("unable to create zstd context");
- size_t in_size = ZSTD_initDStream(self->ds);
- if (ZSTD_isError(in_size)) panic(ZSTD_getErrorName(in_size));
- self->out.size = ZSTD_DStreamOutSize();
- self->out.dst = malloc(self->out.size + ZSTD_DStreamInSize());
- if (self->out.dst == NULL) panic("unable to allocate input buffers");
- self->out.pos = 0;
- self->in.src = (const char *)self->out.dst + self->out.size;
- self->in.size = fread((void *)self->in.src, 1, in_size, self->stream);
- self->in.pos = 0;
- self->pos = 0;
}
static void InputStream_close(struct InputStream *self) {
- free(self->out.dst);
- ZSTD_freeDStream(self->ds);
fclose(self->stream);
}
static bool InputStream_atEnd(struct InputStream *self) {
- while (self->pos >= self->out.pos) {
- self->out.pos = 0;
- self->pos = 0;
- size_t in_size = ZSTD_decompressStream(self->ds, &self->out, &self->in);
- if (ZSTD_isError(in_size)) panic(ZSTD_getErrorName(in_size));
- if (self->in.pos >= self->in.size) {
- size_t max_in_size = ZSTD_DStreamInSize();
- if (in_size > max_in_size) in_size = max_in_size;
- self->in.size = fread((void *)self->in.src, 1, in_size, self->stream);
- self->in.pos = 0;
- if (self->in.pos >= self->in.size) return true;
- }
- }
- return false;
+ return feof(self->stream) != 0;
}
static uint8_t InputStream_readByte(struct InputStream *self) {
- if (InputStream_atEnd(self)) panic("unexpected end of input stream");
- uint8_t value = ((uint8_t *)self->out.dst)[self->pos];
- self->pos += 1;
+ int value;
+ value = fgetc(self->stream);
+ if (value == EOF) panic("unexpected end of input stream");
return value;
}
@@ -168,26 +136,13 @@ static char *InputStream_readName(struct InputStream *self) {
uint32_t len = InputStream_readLeb128_u32(self);
char *name = malloc(len + 1);
if (name == NULL) panic("out of memory");
- for (uint32_t i = 0; i < len; ) {
- if (InputStream_atEnd(self)) panic("unexpected end of input stream");
- size_t remaining = self->out.pos - self->pos;
- if (remaining > len - i) remaining = len - i;
- memcpy(&name[i], &((char *)self->out.dst)[self->pos], remaining);
- i += remaining;
- self->pos += remaining;
- }
- name[len] = '\0';
+ if (fread(name, 1, len, self->stream) != len) panic("unexpected end of input stream");
+ name[len] = 0;
return name;
}
static void InputStream_skipBytes(struct InputStream *self, size_t len) {
- for (size_t i = 0; i < len; ) {
- if (InputStream_atEnd(self)) panic("unexpected end of input stream");
- size_t remaining = self->out.pos - self->pos;
- if (remaining > len - i) remaining = len - i;
- i += remaining;
- self->pos += remaining;
- }
+ if (fseek(self->stream, len, SEEK_CUR) == -1) panic("unexpected end of input stream");
}
static uint32_t InputStream_skipToSection(struct InputStream *self, uint8_t expected_id) {
build.zig
@@ -502,17 +502,8 @@ fn addWasiUpdateStep(b: *Builder, version: [:0]const u8) !void {
run_opt.addArg("-o");
run_opt.addFileSourceArg(.{ .path = "stage1/zig1.wasm" });
- const run_zstd = b.addSystemCommand(&.{ "zstd", "-19", "-f" });
- run_zstd.step.dependOn(&run_opt.step);
- run_zstd.addFileSourceArg(.{ .path = "stage1/zig1.wasm" });
- run_zstd.addArg("-o");
- run_zstd.addFileSourceArg(.{ .path = "stage1/zig1.wasm.zst" });
-
- const cleanup = b.addRemoveDirTree("stage1/zig1.wasm");
- cleanup.step.dependOn(&run_zstd.step);
-
- const update_zig1_step = b.step("update-zig1", "Update stage1/zig1.wasm.zst");
- update_zig1_step.dependOn(&cleanup.step);
+ const update_zig1_step = b.step("update-zig1", "Update stage1/zig1.wasm");
+ update_zig1_step.dependOn(&run_opt.step);
}
fn addCompilerStep(b: *Builder) *std.build.LibExeObjStep {
CMakeLists.txt
@@ -181,16 +181,6 @@ set(ZIG_CONFIG_ZIG_OUT "${CMAKE_BINARY_DIR}/config.zig")
set(ZIG_WASM2C_SOURCES
"${CMAKE_SOURCE_DIR}/stage1/wasm2c.c"
- "${CMAKE_SOURCE_DIR}/stage1/zstd/lib/decompress/huf_decompress.c"
- "${CMAKE_SOURCE_DIR}/stage1/zstd/lib/decompress/zstd_ddict.c"
- "${CMAKE_SOURCE_DIR}/stage1/zstd/lib/decompress/zstd_decompress.c"
- "${CMAKE_SOURCE_DIR}/stage1/zstd/lib/decompress/zstd_decompress_block.c"
- "${CMAKE_SOURCE_DIR}/stage1/zstd/lib/common/entropy_common.c"
- "${CMAKE_SOURCE_DIR}/stage1/zstd/lib/common/error_private.c"
- "${CMAKE_SOURCE_DIR}/stage1/zstd/lib/common/fse_decompress.c"
- "${CMAKE_SOURCE_DIR}/stage1/zstd/lib/common/pool.c"
- "${CMAKE_SOURCE_DIR}/stage1/zstd/lib/common/xxhash.c"
- "${CMAKE_SOURCE_DIR}/stage1/zstd/lib/common/zstd_common.c"
)
set(ZIG_CPP_SOURCES
# These are planned to stay even when we are self-hosted.
@@ -738,21 +728,19 @@ else()
endif()
endif()
-set(ZIG1_WASM_ZST_SOURCE "${CMAKE_SOURCE_DIR}/stage1/zig1.wasm.zst")
+set(ZIG1_WASM_MODULE "${CMAKE_SOURCE_DIR}/stage1/zig1.wasm")
set(ZIG1_C_SOURCE "${CMAKE_BINARY_DIR}/zig1.c")
set(ZIG2_C_SOURCE "${CMAKE_BINARY_DIR}/zig2.c")
set(ZIG_COMPILER_RT_C_SOURCE "${CMAKE_BINARY_DIR}/compiler_rt.c")
add_executable(zig-wasm2c ${ZIG_WASM2C_SOURCES})
set_target_properties(zig-wasm2c PROPERTIES COMPILE_FLAGS "${ZIG_WASM2C_COMPILE_FLAGS}")
-target_include_directories(zig-wasm2c PUBLIC "${CMAKE_SOURCE_DIR}/stage1/zstd/lib")
-target_compile_definitions(zig-wasm2c PRIVATE ZSTD_DISABLE_ASM)
add_custom_command(
OUTPUT "${ZIG1_C_SOURCE}"
- COMMAND zig-wasm2c "${ZIG1_WASM_ZST_SOURCE}" "${ZIG1_C_SOURCE}"
- DEPENDS zig-wasm2c "${ZIG1_WASM_ZST_SOURCE}"
- COMMENT STATUS "Converting ${ZIG1_WASM_ZST_SOURCE} to ${ZIG1_C_SOURCE}"
+ COMMAND zig-wasm2c "${ZIG1_WASM_MODULE}" "${ZIG1_C_SOURCE}"
+ DEPENDS zig-wasm2c "${ZIG1_WASM_MODULE}"
+ COMMENT STATUS "Converting ${ZIG1_WASM_MODULE} to ${ZIG1_C_SOURCE}"
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
)
@@ -764,8 +752,6 @@ if(MSVC)
else()
target_link_libraries(zig1 LINK_PUBLIC m)
endif()
-target_include_directories(zig1 PUBLIC "${CMAKE_SOURCE_DIR}/stage1/zstd/lib")
-target_compile_definitions(zig1 PRIVATE ZSTD_DISABLE_ASM)
set(BUILD_ZIG2_ARGS
"${CMAKE_SOURCE_DIR}/lib"